Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2716988 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.019337436 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.02048056 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.033985682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.115251195Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.116546517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.116675828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.116697135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.116711992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"Crucible 9038c2cc-7aac-47f5-8148-a3bd5750b7de has session id: b33ffafa-b80c-413a-b5fa-eb915b644d24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.117787326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Upstairs opts: Upstairs UUID: 9038c2cc-7aac-47f5-8148-a3bd5750b7de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.117933355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Crucible stats registered with UUID: 9038c2cc-7aac-47f5-8148-a3bd5750b7de","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.117954501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:26.11796744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119212589Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119278439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"9038c2cc-7aac-47f5-8148-a3bd5750b7de active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119324872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.11934227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119391132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.11940676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119420728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119433926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119447684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119471511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119485309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119498567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:59181)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119763278Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:50748)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119801673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:55940)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.119828469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.121477379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.121514973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.121549148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.122619593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.122694222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.122712749Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123537079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123604099Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123711294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123730251Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123744699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123758467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123771985Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123785823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123798831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123811919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123828737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123841705Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123854523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123867041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123976445Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"downstairs","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.123994782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124006941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"0","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124019919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"1","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124032647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124045425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124057493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"9038c2cc-7aac-47f5-8148-a3bd5750b7de is now active with session: b33ffafa-b80c-413a-b5fa-eb915b644d24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124070042Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.12408162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:26.124093538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:30.218439376Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.218575447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:30.218618041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:30.218653835Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.218681441Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b33ffafa-b80c-413a-b5fa-eb915b644d24"} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.249590813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.298817448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.299741494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.299782878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.299798266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.299812813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} {"msg":"Crucible 6cc2dc0d-09e6-4b98-92f7-44922de7aeed has session id: 5bdd170d-eea9-4f46-a944-8e1692603564","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.300572883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Upstairs opts: Upstairs UUID: 6cc2dc0d-09e6-4b98-92f7-44922de7aeed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.300621696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Crucible stats registered with UUID: 6cc2dc0d-09e6-4b98-92f7-44922de7aeed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.300646332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:30.300695735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301215169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301248224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} {"msg":"6cc2dc0d-09e6-4b98-92f7-44922de7aeed active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301306196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301331942Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301354439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301368567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301383485Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301403982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301428028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301448595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301473522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301503227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"ds_connection connected from Ok(127.0.0.1:36641)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301531273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"ds_connection connected from Ok(127.0.0.1:61980)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301708178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"ds_connection connected from Ok(127.0.0.1:50641)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.301740063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.30224284Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.302271166Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.302908523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.302946088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.30361607Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.303903358Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304055066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304190996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304675306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304694373Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304708571Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304725298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304739156Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304752834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304765992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304778991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304792539Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304807377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304828523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304884535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.30491773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B"Wait for a query_work_queue command to finish before sending IO ,"pid":1190,"":"downstairs","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304938947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304951046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"0","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304964034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"1","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.304976742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"client":"2","":"downstairs","session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.3049897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.305005628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"6cc2dc0d-09e6-4b98-92f7-44922de7aeed is now active with session: 5bdd170d-eea9-4f46-a944-8e1692603564","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.305018416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.305030114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190,"session_id":"5bdd170d-eea9-4f46-a944-8e1692603564"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:30.305041982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 24576 data: 2 2 2 2 2 2 02/30 Write block 39 len 12288 data: 2 2 2 03/30 Read block 199 len 16384 04/30 Read block 45 len 40960 05/30 Read block 152 len 40960 06/30 Write block 453 len 20480 data: 2 2 2 2 2 07/30 Read block 279 len 8192 08/30 Read block 433 len 16384 09/30 Read block 273 len 32768 10/30 Read block 529 len 36864 11/30 Flush 12/30 Write block 451 len 40960 data: 2 2 3 3 3 3 3 2 2 2 13/30 Read block 155 len 24576 14/30 Write block 306 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Write block 23 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Flush 17/30 Write block 518 len 20480 data: 2 2 2 2 2 18/30 Flush 19/30 Write block 508 len 8192 data: 2 2 20/30 Read block 229 len 24576 21/30 Write block 435 len 36864 data: 2 2 2 2 2 2 2 2 2 22/30 Read block 47 len 36864 23/30 Read block 303 len 16384 24/30 Read block 237 len 24576 25/30 Read block 165 len 36864 26/30 Read block 103 len 24576 27/30 Write block 338 len 8192 data: 2 2 28/30 Write block 205 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Read block 113 len 20480 30/30 Read block 151 len 36864 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:30.32649918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 2 2 2 3 3 3 T T T 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 4 4 4 T T T 26 520-539 2 2 2 4 4 4 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 5 5 5 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 2 2 2 4 4 4 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.581130921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:34.626151779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.626544592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.658429191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659013206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659048761Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659062359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659076467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} {"msg":"Crucible 8dada11a-f676-4609-9f96-5d9688b5d6d6 has session id: 7ebf21e0-dbfd-41d6-b0f6-26089b86fe07","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659711214Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Upstairs opts: Upstairs UUID: 8dada11a-f676-4609-9f96-5d9688b5d6d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659753019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Crucible stats registered with UUID: 8dada11a-f676-4609-9f96-5d9688b5d6d6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.659776115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:34.659796092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660258625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660284961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} {"msg":"8dada11a-f676-4609-9f96-5d9688b5d6d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660342443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66036086Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660383667Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660402264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660417802Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660453737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660476953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66050062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660523446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"ds_connection connected from Ok(127.0.0.1:33970)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660562311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"ds_connection connected from Ok(127.0.0.1:56368)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660587997Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"ds_connection connected from Ok(127.0.0.1:53981)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.660612683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.661310492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.661338428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.661354306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.661990093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662017799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662036616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662606153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662633169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66269449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662713828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662728475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662742413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662756321Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66276994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662783467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662796856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662810714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662823292Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66283614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662852897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662866155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"":"downstairs","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2025-01-14T16:10:34.662885403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662897511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"0","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662910759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"1","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662923587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"client":"2","":"downstairs","session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662936755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662948823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"8dada11a-f676-4609-9f96-5d9688b5d6d6 is now active with session: 7ebf21e0-dbfd-41d6-b0f6-26089b86fe07","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662961221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.66297293Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198,"session_id":"7ebf21e0-dbfd-41d6-b0f6-26089b86fe07"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:34.662984778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 497 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 69 len 8192 data: 2 2 03/30 Write block 376 len 16384 data: 2 2 2 2 04/30 Read block 159 len 20480 05/30 Read block 5 len 36864 06/30 Read block 446 len 40960 07/30 Flush 08/30 Write block 499 len 8192 data: 3 3 09/30 Read block 177 len 20480 10/30 Write block 256 len 20480 data: 2 2 2 2 2 11/30 Flush 12/30 Write block 95 len 28672 data: 2 2 2 2 2 2 2 13/30 Read block 313 len 24576 14/30 Read block 524 len 12288 15/30 Write block 403 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Read block 488 len 12288 17/30 Flush 18/30 Write block 361 len 12288 data: 2 2 2 19/30 Write block 457 len 40960 data: 4 3 3 3 2 2 2 2 2 2 20/30 Read block 295 len 12288 21/30 Write block 266 len 28672 data: 2 2 2 2 2 2 2 22/30 Write block 130 len 28672 data: 2 2 2 2 2 2 2 23/30 Write block 319 len 4096 data: 2 24/30 Read block 192 len 8192 25/30 Write block 63 len 4096 data: 2 26/30 Read block 34 len 16384 27/30 Write block 425 len 4096 data: 2 28/30 Write block 538 len 12288 data: 2 2 2 29/30 Write block 564 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Write block 269 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:34.680983878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 4 4 4 6 6 6 T T T 4 080-099 4 4 4 8 8 8 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 7 7 7 F F F 13 260-279 4 4 4 7 7 7 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 T T T 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 4 4 4 6 6 6 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 3 3 3 5 5 5 T T T 22 440-459 3 3 3 5 5 5 T T T 23 460-479 2 2 2 3 3 3 T T T 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 2 2 2 4 4 4 T T T 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 8 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 5 5 5 9 9 9 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 7 7 7 F F F 13 260-279 5 5 5 9 9 9 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 9 9 9 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 5 5 5 9 9 9 F F F 22 440-459 5 5 5 9 9 9 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 5 5 5 9 9 9 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:38.981297461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.027125471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.027825899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.027895129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.027915206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.027933434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} {"msg":"Crucible 56adf6fd-1698-4ae2-9965-d0d333257950 has session id: 705f2b4c-dbec-47ef-bc57-a767433d3963","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.028457227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Upstairs opts: Upstairs UUID: 56adf6fd-1698-4ae2-9965-d0d333257950, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.028502361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Crucible stats registered with UUID: 56adf6fd-1698-4ae2-9965-d0d333257950","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.028519938Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:39.028538575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029011647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029036363Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} {"msg":"56adf6fd-1698-4ae2-9965-d0d333257950 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029108093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029136379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029174313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029200659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029228315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029254042Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029283377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029309643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029337659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029364025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"ds_connection connected from Ok(127.0.0.1:34190)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029388652Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"ds_connection connected from Ok(127.0.0.1:33238)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029425956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"ds_connection connected from Ok(127.0.0.1:34882)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.029498456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.0300909Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.030118156Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.030645949Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.030815674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.030855648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031246092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031451591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031475798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 2, 9, 8, 8, 9, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031766106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 5, 4, 4, 5, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031785853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031800311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 2, 9, 8, 8, 9, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031813969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 5, 4, 4, 5, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031827257Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031841125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 2, 9, 8, 8, 9, 1, 1, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031858282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 5, 4, 4, 5, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.03187208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031886028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031898897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:10:39.031912095Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031929262Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.03194199Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"":"downstairs","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031954938Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031966787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"0","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031980065Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"1","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.031992953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"client":"2","":"downstairs","session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.032005921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.032017969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"56adf6fd-1698-4ae2-9965-d0d333257950 is now active with session: 705f2b4c-dbec-47ef-bc57-a767433d3963","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.032030557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.032042146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206,"session_id":"705f2b4c-dbec-47ef-bc57-a767433d3963"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:39.032055834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 356 len 32768 03/30 Write block 283 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 141 len 4096 05/30 Read block 294 len 8192 06/30 Write block 75 len 24576 data: 2 2 2 2 2 2 07/30 Write block 430 len 8192 data: 2 2 08/30 Read block 341 len 8192 09/30 Write block 62 len 16384 data: 2 3 2 2 10/30 Read block 191 len 28672 11/30 Read block 320 len 40960 12/30 Write block 92 len 40960 data: 2 2 2 3 3 3 3 3 3 3 13/30 Flush 14/30 Write block 209 len 16384 data: 3 3 3 3 15/30 Read block 285 len 40960 16/30 Read block 566 len 16384 17/30 Write block 435 len 28672 data: 3 3 3 3 3 3 3 18/30 Read block 416 len 12288 19/30 Flush 20/30 Write block 324 len 36864 data: 2 2 2 2 2 2 2 2 2 21/30 Read block 391 len 36864 22/30 Read block 457 len 36864 23/30 Read block 380 len 32768 24/30 Flush 25/30 Write block 405 len 40960 data: 3 3 3 3 3 3 3 3 2 2 26/30 Write block 383 len 28672 data: 2 2 2 2 2 2 2 27/30 Write block 462 len 8192 data: 3 3 28/30 Read block 410 len 8192 29/30 Read block 87 len 4096 30/30 Read block 247 len 4096 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:39.05255681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 6 6 6 10 10 10 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 4 4 4 7 7 7 F F F 13 260-279 5 5 5 9 9 9 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 12 12 12 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 9 9 9 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 4 4 4 8 8 8 T T T 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 5 5 5 9 9 9 T T T 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 5 5 5 9 9 9 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 12 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 6 6 6 10 10 10 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 4 4 4 7 7 7 F F F 13 260-279 5 5 5 9 9 9 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 12 12 12 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 9 9 9 F F F 19 380-399 7 7 7 13 13 13 F F F 20 400-419 7 7 7 13 13 13 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 7 7 7 13 13 13 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 5 5 5 9 9 9 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 13 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.274474022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.319041575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.319720047Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.319772179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.319790346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.319808314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} {"msg":"Crucible 0240225c-f186-4a5f-8514-2759d4e1a825 has session id: 88fe03af-c446-41fc-b9b2-aab5714487b7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.32038454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Upstairs opts: Upstairs UUID: 0240225c-f186-4a5f-8514-2759d4e1a825, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.320427573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Crucible stats registered with UUID: 0240225c-f186-4a5f-8514-2759d4e1a825","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.32045134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:43.320472437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.320893676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.320920612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} {"msg":"0240225c-f186-4a5f-8514-2759d4e1a825 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.320978653Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.320997091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321019837Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321033555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321047883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321061271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321075409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321088687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321102315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321122672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37144)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321336161Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51870)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.321447065Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37216)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.32148424Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322037159Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322180849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322221992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322607366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322645011Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.322894994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323172574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.32319775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 2, 10, 10, 10, 9, 1, 1, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323489428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 6, 6, 6, 5, 1, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323516234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323531752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 2, 10, 10, 10, 9, 1, 1, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323550079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 6, 6, 6, 5, 1, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323564017Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323578015Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 2, 10, 10, 10, 9, 1, 1, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323591153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 6, 6, 6, 5, 1, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323604471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323618179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323630797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323643585Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323656294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323675361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"":"downstairs","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323688419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323700087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"0","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323712995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"1","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323725564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"client":"2","":"downstairs","session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323741001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.3237531Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"0240225c-f186-4a5f-8514-2759d4e1a825 is now active with session: 88fe03af-c446-41fc-b9b2-aab5714487b7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323765818Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323777616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214,"session_id":"88fe03af-c446-41fc-b9b2-aab5714487b7"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:43.323789484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 61 len 24576 data: 2 3 4 3 3 2 02/30 Read block 187 len 40960 03/30 Write block 144 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Flush 05/30 Write block 365 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 17 len 12288 07/30 Read block 119 len 36864 08/30 Write block 475 len 20480 data: 2 2 2 2 2 09/30 Write block 51 len 32768 data: 3 3 2 2 2 2 2 2 10/30 Flush 11/30 Write block 156 len 40960 data: 2 2 2 2 2 2 2 2 2 2 12/30 Write block 245 len 4096 data: 2 13/30 Write block 335 len 24576 data: 2 2 2 3 3 2 14/30 Write block 271 len 32768 data: 3 3 2 2 2 2 2 2 15/30 Write block 381 len 40960 data: 2 2 3 3 3 3 3 3 3 2 16/30 Write block 432 len 4096 data: 2 17/30 Write block 147 len 12288 data: 3 3 3 18/30 Read block 102 len 4096 19/30 Read block 97 len 8192 20/30 Read block 53 len 36864 21/30 Read block 516 len 24576 22/30 Write block 504 len 16384 data: 3 3 2 2 23/30 Read block 292 len 40960 24/30 Flush 25/30 Write block 468 len 8192 data: 2 2 26/30 Read block 283 len 4096 27/30 Read block 316 len 24576 28/30 Write block 141 len 16384 data: 2 2 2 3 29/30 Write block 356 len 36864 data: 2 2 2 2 2 3 3 3 2 30/30 Write block 541 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:43.342314138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 8 8 8 14 14 14 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 8 8 8 16 16 16 T T T 8 160-179 8 8 8 16 16 16 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 8 8 8 16 16 16 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 8 8 8 16 16 16 F F F 17 340-359 8 8 8 16 16 16 T T T 18 360-379 8 8 8 15 15 15 T T T 19 380-399 8 8 8 16 16 16 F F F 20 400-419 7 7 7 13 13 13 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 8 8 8 15 15 15 T T T 24 480-499 4 4 4 7 7 7 F F F 25 500-519 8 8 8 16 16 16 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 5 5 5 9 9 9 T T T 28 560-579 5 5 5 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 16 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 8 8 8 14 14 14 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 8 8 8 16 16 16 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 8 8 8 16 16 16 F F F 17 340-359 9 9 9 17 17 17 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 8 8 8 16 16 16 F F F 20 400-419 7 7 7 13 13 13 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 8 8 8 16 16 16 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 9 9 9 17 17 17 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.581169414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.625565513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62613647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.626177844Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.626195961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.626210799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} {"msg":"Crucible dce5628d-6bd3-4f0d-9806-71fc393e5c42 has session id: a1fcb70a-9edf-4828-bf3a-27acea01faab","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.626825679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Upstairs opts: Upstairs UUID: dce5628d-6bd3-4f0d-9806-71fc393e5c42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.626868703Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Crucible stats registered with UUID: dce5628d-6bd3-4f0d-9806-71fc393e5c42","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62689514Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:47.626918156Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627424822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627452218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} {"msg":"dce5628d-6bd3-4f0d-9806-71fc393e5c42 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627503711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627529437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627551614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627565762Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62758001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627596537Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627610715Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627623603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627639631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627665567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"ds_connection connected from Ok(127.0.0.1:34369)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627692353Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"ds_connection connected from Ok(127.0.0.1:54291)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627729128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"ds_connection connected from Ok(127.0.0.1:59164)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.627899553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.628496136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.628524142Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.628548759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629132804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629171238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629198414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629734976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629760143Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 15, 14, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62977603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[0]R generation[0..12]: [1, 2, 8, 8, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629790538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629804586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 15, 14, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629818054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[1]R generation[0..12]: [1, 2, 8, 8, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629831872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62984547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 15, 14, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629858858Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[2]R generation[0..12]: [1, 2, 8, 8, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629871806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629885384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629898082Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629910981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629923539Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629936007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"":"downstairs","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.629952255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:10:47.629964573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"0","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.62998133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"1","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.630009296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"client":"2","":"downstairs","session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.630024864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.630037132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"dce5628d-6bd3-4f0d-9806-71fc393e5c42 is now active with session: a1fcb70a-9edf-4828-bf3a-27acea01faab","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.63004965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.630061329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222,"session_id":"a1fcb70a-9edf-4828-bf3a-27acea01faab"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:47.630073347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 32768 data: 3 3 2 2 2 2 2 2 02/30 Read block 571 len 28672 03/30 Write block 337 len 16384 data: 3 4 4 3 04/30 Write block 398 len 36864 data: 2 2 2 2 2 3 3 4 4 05/30 Flush 06/30 Write block 38 len 8192 data: 3 3 07/30 Read block 158 len 12288 08/30 Flush 09/30 Write block 65 len 8192 data: 4 3 10/30 Write block 578 len 16384 data: 2 2 2 2 11/30 Read block 316 len 16384 12/30 Read block 386 len 8192 13/30 Write block 531 len 32768 data: 2 2 2 2 2 2 2 3 14/30 Flush 15/30 Write block 519 len 8192 data: 3 3 16/30 Read block 38 len 4096 17/30 Write block 251 len 32768 data: 2 2 2 2 2 3 3 3 18/30 Write block 358 len 24576 data: 3 3 3 4 4 4 19/30 Write block 388 len 40960 data: 4 4 3 2 2 2 2 2 2 2 20/30 Read block 330 len 40960 21/30 Read block 184 len 36864 22/30 Read block 26 len 20480 23/30 Read block 409 len 24576 24/30 Read block 496 len 36864 25/30 Write block 323 len 28672 data: 2 3 3 3 3 3 3 26/30 Read block 490 len 12288 27/30 Write block 353 len 20480 data: 2 2 2 3 3 28/30 Write block 518 len 24576 data: 3 4 4 3 3 2 29/30 Write block 497 len 12288 data: 3 3 4 30/30 Read block 288 len 32768 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 3 4 Done 16 15 13 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:47.648916644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 10 10 10 19 19 19 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 10 10 10 20 20 20 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 8 8 8 16 16 16 T T T 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 10 10 10 18 18 18 T T T 17 340-359 10 10 10 18 18 18 T T T 18 360-379 9 9 9 17 17 17 T T T 19 380-399 10 10 10 18 18 18 T T T 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 4 4 4 7 7 7 T T T 25 500-519 8 8 8 16 16 16 T T T 26 520-539 10 10 10 20 20 20 T T T 27 540-559 9 9 9 17 17 17 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 10 10 10 20 20 20 F F F Max gen: 10, Max flush: 20 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 10 10 10 19 19 19 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 10 10 10 20 20 20 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 11 11 11 21 21 21 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 11 11 11 21 21 21 F F F 25 500-519 11 11 11 21 21 21 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 9 9 9 17 17 17 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 10 10 10 20 20 20 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.894265315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:51.93779941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.938228137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.970676125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.971304873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.971347937Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.971364445Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.971382042Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} {"msg":"Crucible e477551f-0345-4bbd-98fc-ed0a808568f9 has session id: 206d2b3b-9146-4a29-b7bc-93826c516018","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.971982235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Upstairs opts: Upstairs UUID: e477551f-0345-4bbd-98fc-ed0a808568f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972041496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Crucible stats registered with UUID: e477551f-0345-4bbd-98fc-ed0a808568f9","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972059564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:51.972077411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972484332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972508618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} {"msg":"e477551f-0345-4bbd-98fc-ed0a808568f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972571999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972589016Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972612083Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972626171Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972640149Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972653037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972666645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972682413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972741244Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.97276855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"ds_connection connected from Ok(127.0.0.1:59752)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972807675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"ds_connection connected from Ok(127.0.0.1:62645)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972831651Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"ds_connection connected from Ok(127.0.0.1:62041)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.972862556Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.973546797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.973578163Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.973645933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974159828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974243156Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974315025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974707538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974811683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[0]R flush_numbers[0..12]: [1, 19, 15, 20, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974887172Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[0]R generation[0..12]: [1, 10, 8, 10, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974907339Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974921917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[1]R flush_numbers[0..12]: [1, 19, 15, 20, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974935655Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[1]R generation[0..12]: [1, 10, 8, 10, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974948913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974962501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[2]R flush_numbers[0..12]: [1, 19, 15, 20, 10, 10, 9, 17, 16, 1, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974980418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[2]R generation[0..12]: [1, 10, 8, 10, 6, 6, 5, 9, 8, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.974993986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975007544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975020213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975032931Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975045419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975068986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"":"downstairs","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975082553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975094112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"0","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975113689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"1","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975126607Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"client":"2","":"downstairs","session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975139535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975151514Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"e477551f-0345-4bbd-98fc-ed0a808568f9 is now active with session: 206d2b3b-9146-4a29-b7bc-93826c516018","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975164002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.97517575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230,"session_id":"206d2b3b-9146-4a29-b7bc-93826c516018"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:51.975190228Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 16384 data: 4 5 4 5 02/30 Read block 234 len 16384 03/30 Read block 247 len 32768 04/30 Read block 493 len 12288 05/30 Write block 373 len 20480 data: 2 2 2 3 3 06/30 Flush 07/30 Write block 192 len 24576 data: 2 2 2 2 2 2 08/30 Flush 09/30 Write block 574 len 36864 data: 2 2 2 2 3 3 3 3 2 10/30 Read block 180 len 16384 11/30 Read block 56 len 4096 12/30 Write block 488 len 20480 data: 2 2 2 2 2 13/30 Write block 187 len 40960 data: 2 2 2 2 2 3 3 3 3 3 14/30 Read block 595 len 4096 15/30 Read block 37 len 20480 16/30 Read block 126 len 12288 17/30 Read block 223 len 12288 18/30 Read block 251 len 8192 19/30 Read block 101 len 8192 20/30 Read block 207 len 36864 21/30 Read block 350 len 32768 22/30 Flush 23/30 Flush 24/30 Write block 549 len 28672 data: 2 2 2 2 2 2 2 25/30 Read block 236 len 32768 26/30 Write block 335 len 16384 data: 3 3 4 5 27/30 Read block 450 len 28672 28/30 Write block 505 len 40960 data: 4 3 3 3 3 2 2 2 2 2 29/30 Write block 1 len 20480 data: 2 2 2 2 2 30/30 Write block 62 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:51.994821341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 10 10 10 19 19 19 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 12 12 12 22 22 22 T T T 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 12 12 12 24 24 24 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 11 11 11 21 21 21 T T T 17 340-359 11 11 11 21 21 21 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 12 12 12 24 24 24 F F F 25 500-519 11 11 11 21 21 21 T T T 26 520-539 11 11 11 21 21 21 F F F 27 540-559 9 9 9 17 17 17 T T T 28 560-579 12 12 12 24 24 24 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 12, Max flush: 24 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 10 10 10 19 19 19 F F F 2 040-059 8 8 8 15 15 15 F F F 3 060-079 13 13 13 25 25 25 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 12 12 12 24 24 24 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 12 12 12 24 24 24 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 13 13 25 25 25 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.259601724Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.303958669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.304624822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.304665286Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.304679774Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.304694382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} {"msg":"Crucible 0956a52b-6dd3-40a8-9967-d5cef84d206e has session id: 0b6c5238-eee5-4a44-8d6b-503a3de8281d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305269078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Upstairs opts: Upstairs UUID: 0956a52b-6dd3-40a8-9967-d5cef84d206e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305312951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Crucible stats registered with UUID: 0956a52b-6dd3-40a8-9967-d5cef84d206e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305336688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:56.305357515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305857162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305881079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} {"msg":"0956a52b-6dd3-40a8-9967-d5cef84d206e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.30594345Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305961567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.305984804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306003101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306028218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306053654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.30607825Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306105976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306138111Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306174606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51994)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.30621684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46335)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306247046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33119)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.306271672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307021573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307052418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307140266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307695265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307723611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.307757966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308302807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308328103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[0]R flush_numbers[0..12]: [25, 19, 15, 25, 10, 10, 9, 17, 16, 24, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308353329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[0]R generation[0..12]: [13, 10, 8, 13, 6, 6, 5, 9, 8, 12, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308368037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308385524Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[1]R flush_numbers[0..12]: [25, 19, 15, 25, 10, 10, 9, 17, 16, 24, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308399302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[1]R generation[0..12]: [13, 10, 8, 13, 6, 6, 5, 9, 8, 12, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.3084128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308426558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[2]R flush_numbers[0..12]: [25, 19, 15, 25, 10, 10, 9, 17, 16, 24, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308439787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[2]R generation[0..12]: [13, 10, 8, 13, 6, 6, 5, 9, 8, 12, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308452765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308466273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308478891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308491669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308504307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308516955Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"":"downstairs","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308529714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:10:56.308541262Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"0","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308560469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"1","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308573237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"client":"2","":"downstairs","session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308588235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308600203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"0956a52b-6dd3-40a8-9967-d5cef84d206e is now active with session: 0b6c5238-eee5-4a44-8d6b-503a3de8281d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308612681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.30862506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238,"session_id":"0b6c5238-eee5-4a44-8d6b-503a3de8281d"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:10:56.308637608Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 541 len 24576 data: 3 3 2 2 2 2 02/30 Write block 293 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 246 len 32768 04/30 Write block 295 len 16384 data: 3 3 3 3 05/30 Read block 355 len 32768 06/30 Read block 554 len 24576 07/30 Read block 321 len 4096 08/30 Read block 348 len 36864 09/30 Read block 508 len 4096 10/30 Read block 546 len 32768 11/30 Write block 495 len 12288 data: 2 2 4 12/30 Write block 180 len 40960 data: 2 2 2 2 2 2 2 3 3 3 13/30 Write block 488 len 20480 data: 3 3 3 3 3 14/30 Write block 40 len 24576 data: 3 3 2 2 2 2 15/30 Read block 356 len 20480 16/30 Read block 369 len 4096 17/30 Read block 471 len 8192 18/30 Write block 362 len 40960 data: 5 5 3 3 3 3 3 3 3 3 19/30 Read block 387 len 4096 20/30 Read block 10 len 28672 21/30 Write block 397 len 12288 data: 3 3 3 22/30 Read block 513 len 24576 23/30 Read block 74 len 8192 24/30 Write block 0 len 20480 data: 2 3 3 3 3 25/30 Write block 32 len 32768 data: 4 3 3 3 3 3 4 4 26/30 Write block 501 len 16384 data: 3 3 3 4 27/30 Write block 35 len 36864 data: 4 4 4 5 5 4 4 3 3 28/30 Read block 545 len 12288 29/30 Read block 333 len 20480 30/30 Read block 127 len 4096 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:10:56.327913152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 T T T 1 020-039 10 10 10 19 19 19 T T T 2 040-059 8 8 8 15 15 15 T T T 3 060-079 13 13 13 25 25 25 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 12 12 12 24 24 24 T T T 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 6 6 6 10 10 10 T T T 15 300-319 5 5 5 9 9 9 T T T 16 320-339 13 13 13 25 25 25 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 12 12 12 22 22 22 T T T 19 380-399 11 11 11 21 21 21 T T T 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 12 12 12 24 24 24 T T T 25 500-519 13 13 13 25 25 25 T T T 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 13 13 25 25 25 T T T 28 560-579 12 12 12 24 24 24 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 13, Max flush: 25 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 26 26 26 F F F 1 020-039 14 14 14 26 26 26 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 13 13 13 25 25 25 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 5 5 5 9 9 9 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 14 14 14 26 26 26 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 14 14 14 26 26 26 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 14 14 14 26 26 26 F F F 25 500-519 14 14 14 26 26 26 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 14 14 14 26 26 26 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 14, Max flush: 26 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.59493278Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.639365784Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.639956728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.64000718Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.640025488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.640043755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} {"msg":"Crucible 4e43e58a-ee8d-43c6-87dc-d382634c4fb8 has session id: 3b79e5cf-b941-446d-9197-30fd597612ab","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.640610003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Upstairs opts: Upstairs UUID: 4e43e58a-ee8d-43c6-87dc-d382634c4fb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.640654906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Crucible stats registered with UUID: 4e43e58a-ee8d-43c6-87dc-d382634c4fb8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.640672054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:00.640685532Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641055808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641088983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} {"msg":"4e43e58a-ee8d-43c6-87dc-d382634c4fb8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641147344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.64117464Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641198237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641212785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641226983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641240271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641254449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641267197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641280615Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641302582Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:55348)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641344116Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:00.641387509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:00.641418035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:00.641445831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:49873)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641495944Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.641634754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.642118733Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.642146779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.642738993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"1","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.642880592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.643285893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:00.643368201Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.643078073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:60419)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.643500552Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.644537981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645039598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[0]R flush_numbers[0..12]: [26, 26, 26, 25, 10, 10, 9, 17, 16, 26, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645616884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[0]R generation[0..12]: [14, 14, 14, 13, 6, 6, 5, 9, 8, 14, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645635731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645650199Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[1]R flush_numbers[0..12]: [26, 26, 26, 25, 10, 10, 9, 17, 16, 26, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645669336Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[1]R generation[0..12]: [14, 14, 14, 13, 6, 6, 5, 9, 8, 14, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645683654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645697602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[2]R flush_numbers[0..12]: [26, 26, 26, 25, 10, 10, 9, 17, 16, 26, 11, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.64571145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[2]R generation[0..12]: [14, 14, 14, 13, 6, 6, 5, 9, 8, 14, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645724848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645738966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645752014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Generation requested: 16 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645764952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.64577772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645790448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"":"downstairs","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645803687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645815675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"0","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645828883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"1","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.64584974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"client":"2","":"downstairs","session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645865647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645878465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"4e43e58a-ee8d-43c6-87dc-d382634c4fb8 is now active with session: 3b79e5cf-b941-446d-9197-30fd597612ab","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645891644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.645903842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246,"session_id":"3b79e5cf-b941-446d-9197-30fd597612ab"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:10.64591629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 36864 data: 3 3 3 3 3 2 2 2 2 02/30 Read block 289 len 20480 03/30 Read block 357 len 4096 04/30 Flush 05/30 Write block 428 len 16384 data: 2 2 3 3 06/30 Read block 66 len 24576 07/30 Write block 564 len 4096 data: 3 08/30 Read block 464 len 32768 09/30 Write block 216 len 32768 data: 3 3 3 3 3 3 3 3 10/30 Write block 232 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Read block 131 len 28672 12/30 Read block 444 len 24576 13/30 Read block 327 len 32768 14/30 Read block 356 len 28672 15/30 Write block 208 len 4096 data: 3 16/30 Read block 85 len 28672 17/30 Write block 52 len 24576 data: 4 3 3 3 3 3 18/30 Flush 19/30 Write block 120 len 28672 data: 2 2 2 2 2 2 2 20/30 Flush 21/30 Write block 517 len 12288 data: 2 4 5 22/30 Read block 84 len 36864 23/30 Write block 447 len 36864 data: 2 2 2 2 3 3 4 4 4 24/30 Read block 484 len 20480 25/30 Flush 26/30 Write block 232 len 4096 data: 3 27/30 Write block 415 len 12288 data: 2 2 2 28/30 Write block 491 len 24576 data: 4 4 2 2 3 3 29/30 Read block 190 len 32768 30/30 Write block 397 len 16384 data: 4 4 4 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:10.665418532Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 26 26 26 F F F 1 020-039 14 14 14 26 26 26 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 13 13 13 25 25 25 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 16 16 16 29 29 29 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 16 16 16 28 28 28 F F F 11 220-239 16 16 16 28 28 28 T T T 12 240-259 16 16 16 28 28 28 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 14 14 14 26 26 26 F F F 19 380-399 14 14 14 26 26 26 T T T 20 400-419 10 10 10 18 18 18 T T T 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 14 14 14 26 26 26 T T T 25 500-519 16 16 16 30 30 30 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 14 14 14 26 26 26 F F F 28 560-579 16 16 16 28 28 28 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 16, Max flush: 30 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 26 26 26 F F F 1 020-039 14 14 14 26 26 26 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 13 13 13 25 25 25 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 16 16 16 29 29 29 F F F 7 140-159 9 9 9 17 17 17 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 16 16 16 28 28 28 F F F 11 220-239 17 17 17 31 31 31 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 11 11 11 21 21 21 F F F 18 360-379 14 14 14 26 26 26 F F F 19 380-399 17 17 17 31 31 31 F F F 20 400-419 17 17 17 31 31 31 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 17 17 17 31 31 31 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 14 14 14 26 26 26 F F F 28 560-579 16 16 16 28 28 28 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.905730475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.951373822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.952408552Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.952453565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.952472793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95248781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} {"msg":"Crucible fe40b967-d784-45bf-803d-f96da8fadf98 has session id: aabbdbbc-6a65-49ec-826b-78ae9eaa32ae","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953067756Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Upstairs opts: Upstairs UUID: fe40b967-d784-45bf-803d-f96da8fadf98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953161102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Crucible stats registered with UUID: fe40b967-d784-45bf-803d-f96da8fadf98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953194928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:14.953217044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953690565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953715642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} {"msg":"fe40b967-d784-45bf-803d-f96da8fadf98 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953779003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95379746Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953821106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953835754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953853052Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953877288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953909713Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95393539Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.953961196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954201231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:61348)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954231886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:57771)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954258213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:33796)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954283489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954796494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95483029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.954938474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.955406946Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.955460728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.955707872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956075758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956148218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[0]R flush_numbers[0..12]: [26, 26, 28, 25, 10, 10, 29, 17, 16, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956285618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[0]R generation[0..12]: [14, 14, 16, 13, 6, 6, 16, 9, 8, 14, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956307415Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956322232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[1]R flush_numbers[0..12]: [26, 26, 28, 25, 10, 10, 29, 17, 16, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95633624Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[1]R generation[0..12]: [14, 14, 16, 13, 6, 6, 16, 9, 8, 14, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956350178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956363976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[2]R flush_numbers[0..12]: [26, 26, 28, 25, 10, 10, 29, 17, 16, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956377434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[2]R generation[0..12]: [14, 14, 16, 13, 6, 6, 16, 9, 8, 14, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956390602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95640407Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956416599Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956429417Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956441995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956462072Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"":"downstairs","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95647502Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956490698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"0","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956504386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"1","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956517174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"client":"2","":"downstairs","session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956530002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.95654195Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"fe40b967-d784-45bf-803d-f96da8fadf98 is now active with session: aabbdbbc-6a65-49ec-826b-78ae9eaa32ae","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956554559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956566267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254,"session_id":"aabbdbbc-6a65-49ec-826b-78ae9eaa32ae"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:14.956578115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 16384 data: 4 4 4 4 02/30 Read block 187 len 40960 03/30 Read block 529 len 12288 04/30 Flush 05/30 Write block 123 len 4096 data: 3 06/30 Write block 238 len 8192 data: 3 3 07/30 Read block 218 len 16384 08/30 Read block 500 len 36864 09/30 Write block 414 len 36864 data: 3 3 3 3 2 2 2 2 2 10/30 Write block 173 len 32768 data: 2 2 2 2 2 2 2 3 11/30 Write block 157 len 24576 data: 3 3 3 3 3 3 12/30 Read block 28 len 36864 13/30 Write block 403 len 28672 data: 4 4 5 5 4 4 4 14/30 Read block 44 len 32768 15/30 Write block 546 len 16384 data: 3 2 2 3 16/30 Read block 496 len 32768 17/30 Write block 104 len 28672 data: 2 2 2 2 2 2 2 18/30 Flush 19/30 Write block 572 len 24576 data: 3 2 3 3 3 3 20/30 Write block 184 len 40960 data: 3 3 3 4 4 4 3 3 4 4 21/30 Read block 37 len 28672 22/30 Write block 54 len 4096 data: 4 23/30 Flush 24/30 Write block 338 len 12288 data: 6 5 4 25/30 Read block 409 len 28672 26/30 Write block 381 len 20480 data: 3 3 4 4 4 27/30 Read block 208 len 36864 28/30 Write block 55 len 24576 data: 4 4 4 3 2 2 29/30 Read block 517 len 32768 30/30 Read block 7 len 36864 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:14.976236744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 26 26 26 F F F 1 020-039 14 14 14 26 26 26 F F F 2 040-059 18 18 18 34 34 34 T T T 3 060-079 13 13 13 25 25 25 T T T 4 080-099 6 6 6 10 10 10 F F F 5 100-119 18 18 18 33 33 33 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 18 18 18 33 33 33 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 16 16 16 28 28 28 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 13 13 13 25 25 25 T T T 17 340-359 11 11 11 21 21 21 T T T 18 360-379 14 14 14 26 26 26 F F F 19 380-399 17 17 17 31 31 31 T T T 20 400-419 18 18 18 33 33 33 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 17 17 17 31 31 31 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 18 18 18 33 33 33 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 18, Max flush: 34 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 26 26 26 F F F 1 020-039 14 14 14 26 26 26 F F F 2 040-059 19 19 19 35 35 35 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 18 18 18 33 33 33 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 18 18 18 33 33 33 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 16 16 16 28 28 28 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 19 19 19 35 35 35 F F F 17 340-359 19 19 19 35 35 35 F F F 18 360-379 14 14 14 26 26 26 F F F 19 380-399 19 19 19 35 35 35 F F F 20 400-419 18 18 18 33 33 33 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 17 17 17 31 31 31 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 18 18 18 33 33 33 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 19, Max flush: 35 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.212000381Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.256096313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.256673889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.256716743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.256732081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.256746069Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} {"msg":"Crucible 72393b6d-0108-4d1c-a2df-a8a6358a90e3 has session id: 145e089a-57c0-4d10-9f9c-37327505348c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257256855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Upstairs opts: Upstairs UUID: 72393b6d-0108-4d1c-a2df-a8a6358a90e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257302498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Crucible stats registered with UUID: 72393b6d-0108-4d1c-a2df-a8a6358a90e3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257326414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:19.257344432Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257868545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257893072Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} {"msg":"72393b6d-0108-4d1c-a2df-a8a6358a90e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.257955613Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.25797374Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258000996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258016354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258030362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258049759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258065697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258085534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.25811022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258134897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59210)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258198397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53987)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258268077Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38529)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258304652Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.258910564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.25893703Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.259551501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.259673183Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.26010616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.260178679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.260214834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.260630573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[0]R flush_numbers[0..12]: [26, 26, 35, 35, 10, 33, 33, 33, 33, 34, 28, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261147348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[0]R generation[0..12]: [14, 14, 19, 19, 6, 18, 18, 18, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261169165Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261184593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[1]R flush_numbers[0..12]: [26, 26, 35, 35, 10, 33, 33, 33, 33, 34, 28, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261198531Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[1]R generation[0..12]: [14, 14, 19, 19, 6, 18, 18, 18, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261211929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261225647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[2]R flush_numbers[0..12]: [26, 26, 35, 35, 10, 33, 33, 33, 33, 34, 28, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261238915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[2]R generation[0..12]: [14, 14, 19, 19, 6, 18, 18, 18, 18, 18, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261252223Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261266061Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261283668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261296836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261309705Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261322313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"":"downstairs","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261335121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Transition from WaitQuorum to ActiveWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261354408Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"0","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261367656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"1","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261380364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"client":"2","":"downstairs","session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261393253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261405211Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"72393b6d-0108-4d1c-a2df-a8a6358a90e3 is now active with session: 145e089a-57c0-4d10-9f9c-37327505348c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261417769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261429447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262,"session_id":"145e089a-57c0-4d10-9f9c-37327505348c"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:19.261441326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 484 len 28672 data: 2 2 2 2 4 4 4 02/30 Write block 535 len 32768 data: 3 3 3 4 3 3 4 4 03/30 Read block 167 len 4096 04/30 Read block 547 len 4096 05/30 Read block 277 len 20480 06/30 Write block 329 len 36864 data: 4 3 3 3 2 2 4 4 5 07/30 Flush 08/30 Flush 09/30 Write block 593 len 24576 data: 2 2 2 2 2 2 10/30 Write block 39 len 36864 data: 6 5 5 4 4 3 3 2 3 11/30 Read block 349 len 8192 12/30 Write block 212 len 28672 data: 4 2 2 3 4 4 4 13/30 Write block 17 len 20480 data: 2 2 2 2 2 14/30 Write block 328 len 32768 data: 4 5 4 4 4 3 3 5 15/30 Write block 392 len 40960 data: 3 3 3 3 3 5 5 5 4 3 16/30 Read block 63 len 32768 17/30 Write block 274 len 24576 data: 4 4 4 4 4 3 18/30 Flush 19/30 Write block 447 len 36864 data: 3 3 3 3 4 4 5 5 5 20/30 Write block 355 len 40960 data: 3 4 4 4 4 4 5 6 6 4 21/30 Write block 14 len 8192 data: 2 2 22/30 Write block 584 len 20480 data: 2 2 2 2 2 23/30 Write block 507 len 20480 data: 4 4 4 3 3 24/30 Read block 500 len 4096 25/30 Flush 26/30 Write block 169 len 4096 data: 2 27/30 Write block 277 len 8192 data: 5 5 28/30 Read block 368 len 32768 29/30 Read block 263 len 28672 30/30 Write block 556 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:19.279709077Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 39 39 39 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 18 18 18 33 33 33 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 18 18 18 33 33 33 T T T 9 180-199 18 18 18 34 34 34 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 20 20 20 38 38 38 T T T 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 20 20 20 39 39 39 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 20 20 20 39 39 39 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 36 36 36 T T T 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 20, Max flush: 39 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 39 39 39 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 6 6 6 10 10 10 F F F 5 100-119 18 18 18 33 33 33 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 21 21 21 40 40 40 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 20 20 20 39 39 39 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 20 20 20 39 39 39 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 21, Max flush: 40 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.506857877Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:23.55074659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.551124625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.583771443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.584335181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.584371546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.584385554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.584400922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} {"msg":"Crucible a8dd8c8b-ad46-4b02-9f93-ddc1cb7a745c has session id: 5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585084063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Upstairs opts: Upstairs UUID: a8dd8c8b-ad46-4b02-9f93-ddc1cb7a745c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585119197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Crucible stats registered with UUID: a8dd8c8b-ad46-4b02-9f93-ddc1cb7a745c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585139544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:23.585153242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585557324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.58558094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} {"msg":"a8dd8c8b-ad46-4b02-9f93-ddc1cb7a745c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585633392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585659139Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585682745Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585697673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585712381Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585725829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585740237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585753595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585818126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585842512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58559)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585867518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46287)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585898324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64055)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.585938178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.586618169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.586646475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.586797483Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.587345074Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.587382088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.587410224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.587944286Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.587978741Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[0]R flush_numbers[0..12]: [39, 38, 38, 35, 10, 33, 33, 33, 40, 34, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588001928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[0]R generation[0..12]: [20, 20, 20, 19, 6, 18, 18, 18, 21, 18, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588018265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588032753Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[1]R flush_numbers[0..12]: [39, 38, 38, 35, 10, 33, 33, 33, 40, 34, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588045952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[1]R generation[0..12]: [20, 20, 20, 19, 6, 18, 18, 18, 21, 18, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588062509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588076467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[2]R flush_numbers[0..12]: [39, 38, 38, 35, 10, 33, 33, 33, 40, 34, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588089555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[2]R generation[0..12]: [20, 20, 20, 19, 6, 18, 18, 18, 21, 18, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588102763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588116291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588128689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588141398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588154546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588176852Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"":"downstairs","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588206258Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588226125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"0","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588240073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"1","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588252811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"client":"2","":"downstairs","session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588265629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588277468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"a8dd8c8b-ad46-4b02-9f93-ddc1cb7a745c is now active with session: 5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588293136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588305204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270,"session_id":"5fdfc7a0-7579-4e9d-aa75-9d8f8dbfe3f7"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:23.588317122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 95 len 36864 data: 4 4 4 4 4 4 4 2 2 03/30 Read block 187 len 12288 04/30 Read block 232 len 8192 05/30 Write block 160 len 28672 data: 4 4 4 3 3 3 2 06/30 Read block 281 len 24576 07/30 Flush 08/30 Flush 09/30 Write block 352 len 28672 data: 2 3 3 4 5 5 5 10/30 Write block 184 len 12288 data: 4 4 4 11/30 Flush 12/30 Write block 0 len 36864 data: 3 4 4 4 4 3 2 2 2 13/30 Read block 358 len 24576 14/30 Write block 287 len 40960 data: 3 3 3 2 2 2 3 3 4 4 15/30 Read block 436 len 32768 16/30 Read block 155 len 12288 17/30 Read block 115 len 16384 18/30 Read block 231 len 36864 19/30 Write block 353 len 28672 data: 4 4 5 6 6 6 5 20/30 Read block 433 len 8192 21/30 Read block 79 len 16384 22/30 Write block 86 len 28672 data: 2 2 2 2 2 2 3 23/30 Read block 63 len 8192 24/30 Write block 410 len 40960 data: 4 4 4 3 4 4 4 4 3 3 25/30 Read block 145 len 12288 26/30 Read block 256 len 20480 27/30 Write block 512 len 28672 data: 3 3 3 2 2 3 5 28/30 Read block 110 len 8192 29/30 Read block 257 len 28672 30/30 Write block 285 len 20480 data: 3 3 4 4 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:23.609276052Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 39 39 39 T T T 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 22 22 22 42 42 42 T T T 5 100-119 22 22 22 42 42 42 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 22 22 22 42 42 42 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 21 21 21 40 40 40 F F F 14 280-299 16 16 16 27 27 27 T T T 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 22 22 22 44 44 44 T T T 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 20 20 20 38 38 38 T T T 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 20 20 20 39 39 39 T T T 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 22, Max flush: 44 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 45 45 45 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 23 23 23 45 45 45 F F F 5 100-119 22 22 22 42 42 42 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 22 22 22 42 42 42 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 21 21 21 40 40 40 F F F 14 280-299 23 23 23 45 45 45 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 23 23 23 45 45 45 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 9 9 9 17 17 17 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 23 23 23 45 45 45 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 23, Max flush: 45 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.879474977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.923209901Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.923796586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.92384097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.923857067Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.923872325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} {"msg":"Crucible 1f7a8da5-9aca-43d2-8e42-d3ab931c875b has session id: bcac0106-be80-4b4e-aa49-4106866c2433","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.924487866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Upstairs opts: Upstairs UUID: 1f7a8da5-9aca-43d2-8e42-d3ab931c875b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.924531879Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Crucible stats registered with UUID: 1f7a8da5-9aca-43d2-8e42-d3ab931c875b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.924555626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:27.924577633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.924984203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925011529Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} {"msg":"1f7a8da5-9aca-43d2-8e42-d3ab931c875b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925096897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925122393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.92514593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925161528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925190953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925216789Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925243496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925268832Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925294578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925310316Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"ds_connection connected from Ok(127.0.0.1:42391)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925408601Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"ds_connection connected from Ok(127.0.0.1:35838)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925532434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"ds_connection connected from Ok(127.0.0.1:36609)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.925606383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.926202786Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.92624036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.926975943Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.927038684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.927218148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.927525093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.927691309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.927852036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[0]R flush_numbers[0..12]: [45, 38, 38, 35, 45, 42, 33, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928357512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[0]R generation[0..12]: [23, 20, 20, 19, 23, 22, 18, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928384698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928400716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[1]R flush_numbers[0..12]: [45, 38, 38, 35, 45, 42, 33, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928414814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[1]R generation[0..12]: [23, 20, 20, 19, 23, 22, 18, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928428562Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.92844236Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[2]R flush_numbers[0..12]: [45, 38, 38, 35, 45, 42, 33, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928455648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[2]R generation[0..12]: [23, 20, 20, 19, 23, 22, 18, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928468726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928482334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928495112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.92850811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:11:27.928520728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928589449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"":"downstairs","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928610186Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928625533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"0","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928642811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"1","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928659908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"client":"2","":"downstairs","session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928676726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928692263Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"1f7a8da5-9aca-43d2-8e42-d3ab931c875b is now active with session: bcac0106-be80-4b4e-aa49-4106866c2433","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928708021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928722559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278,"session_id":"bcac0106-be80-4b4e-aa49-4106866c2433"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:27.928737537Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 280 len 40960 data: 3 3 3 3 3 4 4 5 5 5 02/30 Read block 225 len 4096 03/30 Flush 04/30 Write block 283 len 24576 data: 4 4 5 5 6 6 05/30 Write block 281 len 40960 data: 4 4 5 5 6 6 7 7 6 3 06/30 Read block 562 len 20480 07/30 Read block 525 len 4096 08/30 Read block 248 len 20480 09/30 Read block 451 len 16384 10/30 Read block 353 len 12288 11/30 Flush 12/30 Write block 513 len 20480 data: 4 4 3 3 4 13/30 Write block 503 len 32768 data: 4 5 5 4 5 5 5 4 14/30 Read block 32 len 8192 15/30 Read block 531 len 28672 16/30 Read block 185 len 16384 17/30 Read block 458 len 8192 18/30 Read block 577 len 12288 19/30 Write block 252 len 20480 data: 3 3 3 3 4 20/30 Read block 250 len 16384 21/30 Write block 253 len 16384 data: 4 4 4 5 22/30 Write block 8 len 32768 data: 3 2 2 2 2 2 3 3 23/30 Write block 515 len 20480 data: 4 4 5 6 6 24/30 Read block 22 len 28672 25/30 Read block 222 len 28672 26/30 Write block 120 len 20480 data: 3 3 3 4 3 27/30 Read block 528 len 16384 28/30 Read block 538 len 12288 29/30 Write block 463 len 8192 data: 4 3 30/30 Read block 214 len 28672 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 19 17 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:27.947903487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 45 45 45 T T T 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 23 23 23 45 45 45 F F F 5 100-119 22 22 22 42 42 42 F F F 6 120-139 18 18 18 33 33 33 T T T 7 140-159 18 18 18 33 33 33 F F F 8 160-179 22 22 22 42 42 42 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 16 16 16 28 28 28 T T T 13 260-279 21 21 21 40 40 40 F F F 14 280-299 24 24 24 47 47 47 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 23 23 23 45 45 45 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 9 9 9 17 17 17 T T T 24 480-499 20 20 20 36 36 36 F F F 25 500-519 23 23 23 45 45 45 T T T 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 24, Max flush: 47 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 38 38 38 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 23 23 23 45 45 45 F F F 5 100-119 22 22 22 42 42 42 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 22 22 22 42 42 42 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 20 20 20 38 38 38 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 25 25 25 48 48 48 F F F 13 260-279 21 21 21 40 40 40 F F F 14 280-299 24 24 24 47 47 47 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 23 23 23 45 45 45 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 25, Max flush: 48 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.183820872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.228735805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.229548087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.229589691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.229604359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.229619666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} {"msg":"Crucible 7c9a663e-fead-45eb-9406-e75fd8183874 has session id: 2abd0320-dd5b-4049-9d0f-d7f1b01682de","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230085959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Upstairs opts: Upstairs UUID: 7c9a663e-fead-45eb-9406-e75fd8183874, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230124413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Crucible stats registered with UUID: 7c9a663e-fead-45eb-9406-e75fd8183874","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230140451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:32.230161158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230526025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230549271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} {"msg":"7c9a663e-fead-45eb-9406-e75fd8183874 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230611642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.23062902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230651356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230665554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230679862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.23069269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230706498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230734764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230749332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.23076284Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"ds_connection connected from Ok(127.0.0.1:33980)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230787496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"ds_connection connected from Ok(127.0.0.1:41921)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.230951113Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"ds_connection connected from Ok(127.0.0.1:36363)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.231075225Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.231566103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.231728999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.231774053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232148778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232178754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232432777Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232684271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232781806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[0]R flush_numbers[0..12]: [48, 38, 38, 35, 45, 42, 48, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232919866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[0]R generation[0..12]: [25, 20, 20, 19, 23, 22, 25, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232939773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232954641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[1]R flush_numbers[0..12]: [48, 38, 38, 35, 45, 42, 48, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232968559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[1]R generation[0..12]: [25, 20, 20, 19, 23, 22, 25, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.232982357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233010403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[2]R flush_numbers[0..12]: [48, 38, 38, 35, 45, 42, 48, 33, 42, 44, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.23302983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[2]R generation[0..12]: [25, 20, 20, 19, 23, 22, 25, 18, 22, 22, 20, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233044008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233057906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233070744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233083622Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233096351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-14T16:11:32.233108949Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"":"downstairs","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233133615Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233145483Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"0","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233158611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"1","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.2331715Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"client":"2","":"downstairs","session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233184708Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233196566Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"7c9a663e-fead-45eb-9406-e75fd8183874 is now active with session: 2abd0320-dd5b-4049-9d0f-d7f1b01682de","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233209154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.233220833Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286,"session_id":"2abd0320-dd5b-4049-9d0f-d7f1b01682de"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:32.23323498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 32768 data: 4 4 4 4 5 6 5 5 02/30 Write block 32 len 12288 data: 5 4 4 03/30 Read block 328 len 4096 04/30 Write block 36 len 8192 data: 5 5 05/30 Read block 445 len 20480 06/30 Write block 304 len 8192 data: 2 2 07/30 Flush 08/30 Write block 299 len 24576 data: 3 3 3 2 2 3 09/30 Read block 216 len 12288 10/30 Read block 24 len 24576 11/30 Write block 32 len 4096 data: 6 12/30 Read block 315 len 32768 13/30 Read block 16 len 36864 14/30 Write block 92 len 32768 data: 4 3 3 5 5 5 5 5 15/30 Write block 46 len 20480 data: 3 4 3 3 3 16/30 Write block 236 len 20480 data: 3 3 4 4 3 17/30 Write block 164 len 24576 data: 4 4 3 2 2 3 18/30 Read block 202 len 8192 19/30 Write block 556 len 4096 data: 3 20/30 Write block 240 len 24576 data: 4 2 2 2 2 3 21/30 Read block 399 len 16384 22/30 Read block 263 len 8192 23/30 Read block 511 len 36864 24/30 Read block 279 len 4096 25/30 Read block 161 len 36864 26/30 Write block 349 len 16384 data: 2 2 2 3 27/30 Read block 250 len 12288 28/30 Write block 216 len 8192 data: 5 5 29/30 Read block 412 len 28672 30/30 Read block 189 len 8192 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:32.250916817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 26 26 26 49 49 49 T T T 2 040-059 20 20 20 38 38 38 T T T 3 060-079 19 19 19 35 35 35 F F F 4 080-099 23 23 23 45 45 45 T T T 5 100-119 22 22 22 42 42 42 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 22 22 22 42 42 42 T T T 9 180-199 22 22 22 44 44 44 F F F 10 200-219 20 20 20 38 38 38 T T T 11 220-239 18 18 18 33 33 33 T T T 12 240-259 25 25 25 48 48 48 T T T 13 260-279 21 21 21 40 40 40 F F F 14 280-299 24 24 24 47 47 47 T T T 15 300-319 26 26 26 49 49 49 T T T 16 320-339 26 26 26 49 49 49 F F F 17 340-359 23 23 23 45 45 45 T T T 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 40 40 40 T T T 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 26, Max flush: 49 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 27 27 27 50 50 50 F F F 2 040-059 27 27 27 50 50 50 F F F 3 060-079 19 19 19 35 35 35 F F F 4 080-099 27 27 27 50 50 50 F F F 5 100-119 22 22 22 42 42 42 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 27 27 27 50 50 50 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 27 27 27 50 50 50 F F F 12 240-259 27 27 27 50 50 50 F F F 13 260-279 21 21 21 40 40 40 F F F 14 280-299 27 27 27 50 50 50 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 27 27 27 50 50 50 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 27 27 27 50 50 50 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 27, Max flush: 50 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.518944069Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:36.562448617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.562831451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.595403261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.595983766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.596021551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.596037469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.596054826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} {"msg":"Crucible 964c6641-37da-4200-ac20-92707904c8ad has session id: a659e8c6-c407-44cc-937b-2b2e8a9bd69e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.596683425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Upstairs opts: Upstairs UUID: 964c6641-37da-4200-ac20-92707904c8ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.596763743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Crucible stats registered with UUID: 964c6641-37da-4200-ac20-92707904c8ad","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.59678419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:36.596797538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597188581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597211218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} {"msg":"964c6641-37da-4200-ac20-92707904c8ad active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597289447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597307554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597329801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597350938Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597383653Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597410869Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597438975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597467071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597494187Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597522623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58410)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597551668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64552)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597577075Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35750)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.597601461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.598212172Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.598387897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.598437089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.598991429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599027834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.59911807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599594961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599621147Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[0]R flush_numbers[0..12]: [48, 50, 50, 35, 50, 42, 48, 33, 50, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599649273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[0]R generation[0..12]: [25, 27, 27, 19, 27, 22, 25, 18, 27, 22, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599664561Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599679229Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[1]R flush_numbers[0..12]: [48, 50, 50, 35, 50, 42, 48, 33, 50, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599693267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[1]R generation[0..12]: [25, 27, 27, 19, 27, 22, 25, 18, 27, 22, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599706765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599720433Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[2]R flush_numbers[0..12]: [48, 50, 50, 35, 50, 42, 48, 33, 50, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599733701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[2]R generation[0..12]: [25, 27, 27, 19, 27, 22, 25, 18, 27, 22, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599747029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599760697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599773395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599786213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599801991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599814999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"":"downstairs","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599827697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-14T16:11:36.599839325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"0","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599860162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"1","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599873231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"client":"2","":"downstairs","session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599886249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599898217Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"964c6641-37da-4200-ac20-92707904c8ad is now active with session: a659e8c6-c407-44cc-937b-2b2e8a9bd69e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599910635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599922443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294,"session_id":"a659e8c6-c407-44cc-937b-2b2e8a9bd69e"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:36.599934502Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 20480 data: 5 5 4 5 5 02/30 Read block 558 len 16384 03/30 Write block 223 len 20480 data: 4 2 2 2 2 04/30 Read block 549 len 40960 05/30 Read block 404 len 16384 06/30 Write block 39 len 20480 data: 7 6 6 5 5 07/30 Read block 370 len 16384 08/30 Write block 100 len 20480 data: 5 5 3 3 3 09/30 Read block 197 len 24576 10/30 Read block 53 len 40960 11/30 Read block 561 len 40960 12/30 Write block 73 len 20480 data: 2 2 3 3 3 13/30 Flush 14/30 Write block 548 len 4096 data: 3 15/30 Read block 469 len 36864 16/30 Read block 301 len 12288 17/30 Read block 255 len 32768 18/30 Read block 570 len 36864 19/30 Write block 75 len 36864 data: 4 4 4 3 3 3 2 2 2 20/30 Write block 382 len 4096 data: 4 21/30 Write block 257 len 32768 data: 4 4 3 3 2 2 2 2 22/30 Write block 593 len 28672 data: 3 3 3 3 3 3 2 23/30 Read block 370 len 16384 24/30 Read block 524 len 32768 25/30 Write block 110 len 32768 data: 3 2 2 2 2 2 2 2 26/30 Write block 246 len 28672 data: 2 2 2 2 2 3 4 27/30 Write block 95 len 8192 data: 6 6 28/30 Read block 233 len 4096 29/30 Read block 569 len 16384 30/30 Read block 378 len 36864 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:36.620109006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 28 28 28 51 51 51 T T T 4 080-099 27 27 27 50 50 50 T T T 5 100-119 28 28 28 51 51 51 T T T 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 50 50 50 T T T 13 260-279 21 21 21 40 40 40 T T T 14 280-299 27 27 27 50 50 50 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 27 27 27 50 50 50 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 38 38 38 T T T 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 27 27 27 50 50 50 T T T 28 560-579 18 18 18 34 34 34 F F F 29 580-599 20 20 20 39 39 39 T T T Max gen: 28, Max flush: 51 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 29 29 29 52 52 52 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 29 29 29 52 52 52 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 28 28 28 51 51 51 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 29 29 29 52 52 52 F F F 13 260-279 29 29 29 52 52 52 F F F 14 280-299 27 27 27 50 50 50 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 27 27 27 50 50 50 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 29 29 29 52 52 52 F F F 20 400-419 23 23 23 45 45 45 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 18 18 18 34 34 34 F F F 29 580-599 29 29 29 52 52 52 F F F Max gen: 29, Max flush: 52 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.908885687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.952558891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953186059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953228203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953243121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953257569Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} {"msg":"Crucible d4eff746-3d8a-4245-ab17-03c645a7d771 has session id: 431880db-d21b-422a-8706-9e2c9f28987e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953899806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Upstairs opts: Upstairs UUID: d4eff746-3d8a-4245-ab17-03c645a7d771, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.9539393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Crucible stats registered with UUID: d4eff746-3d8a-4245-ab17-03c645a7d771","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.953955587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:40.953972975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954444107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954465993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} {"msg":"d4eff746-3d8a-4245-ab17-03c645a7d771 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954536103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.95455338Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954575337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954588645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954602493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954615391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.95462916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954643407Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954666904Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.9546927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56256)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954801924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34772)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.954837289Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:40.954872554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:40.95490332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:40.954930486Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.955006205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.955467727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.955495554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.956162616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.956189782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.956706207Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:40.956733423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.955973593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52307)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.956105104Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.957158761Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.957688534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[0]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 51, 44, 50, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958247862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[0]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 28, 22, 27, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958269649Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958284297Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[1]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 51, 44, 50, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958298025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[1]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 28, 22, 27, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958311623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958325371Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[2]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 51, 44, 50, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958338749Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[2]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 28, 22, 27, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958352117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958374434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958396251Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958419177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958443784Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958458882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"":"downstairs","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.95847194Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958491427Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"0","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958523192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"1","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958548729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"client":"2","":"downstairs","session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958571805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958584833Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"d4eff746-3d8a-4245-ab17-03c645a7d771 is now active with session: 431880db-d21b-422a-8706-9e2c9f28987e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958599181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958620178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302,"session_id":"431880db-d21b-422a-8706-9e2c9f28987e"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:50.958642575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 24576 data: 5 3 3 3 3 4 02/30 Read block 61 len 24576 03/30 Write block 588 len 20480 data: 3 2 2 2 2 04/30 Flush 05/30 Write block 172 len 32768 data: 2 3 3 3 3 3 3 3 06/30 Read block 38 len 28672 07/30 Read block 506 len 24576 08/30 Write block 480 len 40960 data: 2 2 2 2 3 3 3 3 5 5 09/30 Write block 329 len 4096 data: 7 10/30 Read block 492 len 8192 11/30 Write block 573 len 12288 data: 3 4 4 12/30 Flush 13/30 Write block 342 len 32768 data: 2 2 2 2 2 2 2 3 14/30 Read block 298 len 20480 15/30 Read block 490 len 20480 16/30 Read block 356 len 40960 17/30 Write block 407 len 32768 data: 5 5 5 5 5 5 4 5 18/30 Write block 289 len 28672 data: 7 4 3 3 4 4 5 19/30 Read block 200 len 28672 20/30 Read block 451 len 16384 21/30 Write block 332 len 8192 data: 5 4 22/30 Read block 499 len 32768 23/30 Write block 226 len 8192 data: 3 3 24/30 Write block 290 len 8192 data: 5 4 25/30 Write block 563 len 8192 data: 2 4 26/30 Write block 246 len 24576 data: 3 3 3 3 3 4 27/30 Write block 397 len 40960 data: 6 6 6 5 4 3 5 5 6 6 28/30 Write block 352 len 24576 data: 4 5 5 6 7 7 29/30 Read block 445 len 24576 30/30 Read block 4 len 32768 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:50.978049691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 29 29 29 52 52 52 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 29 29 29 52 52 52 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 28 28 28 51 51 51 T T T 12 240-259 30 30 30 53 53 53 T T T 13 260-279 29 29 29 52 52 52 F F F 14 280-299 27 27 27 50 50 50 T T T 15 300-319 27 27 27 50 50 50 F F F 16 320-339 30 30 30 54 54 54 T T T 17 340-359 27 27 27 50 50 50 T T T 18 360-379 20 20 20 39 39 39 F F F 19 380-399 29 29 29 52 52 52 T T T 20 400-419 23 23 23 45 45 45 T T T 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 30 30 30 54 54 54 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 30 30 30 54 54 54 T T T 29 580-599 30 30 30 53 53 53 F F F Max gen: 30, Max flush: 54 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 28 28 28 51 51 51 F F F 3 060-079 29 29 29 52 52 52 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 29 29 29 52 52 52 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 31 31 31 55 55 55 F F F 12 240-259 31 31 31 55 55 55 F F F 13 260-279 29 29 29 52 52 52 F F F 14 280-299 31 31 31 55 55 55 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 31 31 31 55 55 55 F F F 17 340-359 31 31 31 55 55 55 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 31 31 31 55 55 55 F F F 20 400-419 31 31 31 55 55 55 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 30 30 30 54 54 54 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 31 31 31 55 55 55 F F F 29 580-599 30 30 30 53 53 53 F F F Max gen: 31, Max flush: 55 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.233676576Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.278206196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.278898625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.278968315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.278992091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.279010409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} {"msg":"Crucible dcac3cea-04b8-4ac8-9e2b-ca52c5e7a2b1 has session id: 15cbc3b7-4a5f-4f8e-a875-2143205487b2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.279616501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Upstairs opts: Upstairs UUID: dcac3cea-04b8-4ac8-9e2b-ca52c5e7a2b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.279670123Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Crucible stats registered with UUID: dcac3cea-04b8-4ac8-9e2b-ca52c5e7a2b1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.279690899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:55.279706757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280085932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280118157Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} {"msg":"dcac3cea-04b8-4ac8-9e2b-ca52c5e7a2b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280175889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280202205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280232111Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280258927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280287093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280313449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280341975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280369041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280395327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280421533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60741)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280462917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43824)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280503461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42897)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.280532127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.281153977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.281192831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.281366496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28182055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.281850055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28195936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282346523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282443719Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[0]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 54, 44, 50, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282552353Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[0]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 30, 22, 27, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28257273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282587418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[1]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 54, 44, 50, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282601106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[1]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 30, 22, 27, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282614404Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282627822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[2]R flush_numbers[0..12]: [48, 51, 51, 52, 52, 52, 48, 33, 54, 44, 50, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28264118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[2]R generation[0..12]: [25, 28, 28, 29, 29, 29, 25, 18, 30, 22, 27, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282654278Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282672156Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282685464Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282698402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28271084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282731137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"":"downstairs","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282744115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282755864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"0","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282768782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"1","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.28278153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"client":"2","":"downstairs","session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282794468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282806386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"dcac3cea-04b8-4ac8-9e2b-ca52c5e7a2b1 is now active with session: 15cbc3b7-4a5f-4f8e-a875-2143205487b2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282818854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282830513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310,"session_id":"15cbc3b7-4a5f-4f8e-a875-2143205487b2"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:55.282842281Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 36864 data: 2 3 3 3 3 3 3 3 4 02/30 Write block 220 len 8192 data: 4 4 03/30 Read block 342 len 36864 04/30 Write block 64 len 8192 data: 5 6 05/30 Write block 289 len 32768 data: 8 6 5 4 5 5 6 5 06/30 Write block 37 len 16384 data: 6 6 8 7 07/30 Read block 438 len 16384 08/30 Write block 46 len 32768 data: 4 5 4 4 4 4 5 4 09/30 Write block 107 len 32768 data: 3 3 3 4 3 3 3 3 10/30 Read block 103 len 24576 11/30 Read block 99 len 16384 12/30 Read block 260 len 12288 13/30 Read block 537 len 28672 14/30 Write block 480 len 16384 data: 3 3 3 3 15/30 Read block 439 len 32768 16/30 Read block 428 len 16384 17/30 Write block 336 len 28672 data: 5 6 7 6 5 3 4 18/30 Read block 182 len 12288 19/30 Write block 385 len 24576 data: 5 4 4 5 5 4 20/30 Read block 241 len 16384 21/30 Read block 274 len 40960 22/30 Flush 23/30 Write block 232 len 36864 data: 4 3 3 3 4 4 5 5 6 24/30 Read block 421 len 8192 25/30 Write block 322 len 20480 data: 2 3 5 5 5 26/30 Write block 242 len 36864 data: 4 4 4 5 4 4 4 4 4 27/30 Read block 417 len 20480 28/30 Write block 98 len 32768 data: 6 6 6 6 4 4 4 3 29/30 Read block 234 len 12288 30/30 Read block 482 len 8192 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:55.303336458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 29 29 29 52 52 52 T T T 5 100-119 32 32 32 56 56 56 T T T 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 32 32 32 56 56 56 T T T 12 240-259 31 31 31 55 55 55 T T T 13 260-279 29 29 29 52 52 52 F F F 14 280-299 32 32 32 56 56 56 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 32 32 32 56 56 56 T T T 17 340-359 32 32 32 56 56 56 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 31 31 31 55 55 55 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 32 32 32 56 56 56 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 31 31 31 55 55 55 F F F 29 580-599 30 30 30 53 53 53 F F F Max gen: 32, Max flush: 56 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 33 33 33 57 57 57 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 33 33 33 57 57 57 F F F 12 240-259 33 33 33 57 57 57 F F F 13 260-279 29 29 29 52 52 52 F F F 14 280-299 32 32 32 56 56 56 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 33 33 33 57 57 57 F F F 17 340-359 32 32 32 56 56 56 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 31 31 31 55 55 55 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 F F F 24 480-499 32 32 32 56 56 56 F F F 25 500-519 25 25 25 48 48 48 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 31 31 31 55 55 55 F F F 29 580-599 30 30 30 53 53 53 F F F Max gen: 33, Max flush: 57 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.538452819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.582734125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.583343526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.583395479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.583414046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.583433763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} {"msg":"Crucible de198f6d-2874-435b-9294-3a55ca9914aa has session id: dc8f2ad0-8328-4471-a941-d40f3138297c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.583929271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Upstairs opts: Upstairs UUID: de198f6d-2874-435b-9294-3a55ca9914aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584037035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Crucible stats registered with UUID: de198f6d-2874-435b-9294-3a55ca9914aa","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.58407281Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:59.584095467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584539212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584572387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} {"msg":"de198f6d-2874-435b-9294-3a55ca9914aa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584659185Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584695809Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584736034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584766529Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584793345Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584820471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584848867Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584870074Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584892521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584918567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64699)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584954752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36284)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.584985177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52905)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.585010634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.585631323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.585668948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.585873528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.586257452Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.586287428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.586518544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.586785595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.586877682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[0]R flush_numbers[0..12]: [48, 56, 56, 56, 57, 57, 48, 33, 54, 44, 50, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587042658Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[0]R generation[0..12]: [25, 32, 32, 32, 33, 33, 25, 18, 30, 22, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587071144Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587087391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[1]R flush_numbers[0..12]: [48, 56, 56, 56, 57, 57, 48, 33, 54, 44, 50, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587101909Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[1]R generation[0..12]: [25, 32, 32, 32, 33, 33, 25, 18, 30, 22, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587119887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587134385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[2]R flush_numbers[0..12]: [48, 56, 56, 56, 57, 57, 48, 33, 54, 44, 50, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587148092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[2]R generation[0..12]: [25, 32, 32, 32, 33, 33, 25, 18, 30, 22, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.58716151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587175559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:11:59.587188197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587206704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587219182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.58723179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"":"downstairs","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587244518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587256347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"0","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587270455Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"1","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587283523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"client":"2","":"downstairs","session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587296521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587308559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"de198f6d-2874-435b-9294-3a55ca9914aa is now active with session: dc8f2ad0-8328-4471-a941-d40f3138297c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587323967Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587335635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318,"session_id":"dc8f2ad0-8328-4471-a941-d40f3138297c"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:11:59.587348513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 524 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 331 len 36864 data: 6 6 5 4 6 6 7 8 7 03/30 Write block 475 len 32768 data: 3 3 3 3 3 4 4 4 04/30 Write block 596 len 12288 data: 4 4 4 05/30 Write block 285 len 32768 data: 7 7 8 8 9 7 6 5 06/30 Read block 265 len 20480 07/30 Read block 497 len 16384 08/30 Read block 140 len 24576 09/30 Write block 273 len 40960 data: 3 5 5 5 6 6 4 4 5 5 10/30 Read block 367 len 8192 11/30 Write block 83 len 24576 data: 3 2 2 3 3 3 12/30 Write block 506 len 24576 data: 5 6 6 6 5 4 13/30 Read block 177 len 16384 14/30 Read block 254 len 36864 15/30 Write block 263 len 4096 data: 3 16/30 Write block 501 len 32768 data: 4 4 5 6 6 6 7 7 17/30 Read block 40 len 28672 18/30 Read block 368 len 24576 19/30 Write block 577 len 4096 data: 4 20/30 Read block 403 len 24576 21/30 Write block 142 len 16384 data: 3 3 4 3 22/30 Write block 263 len 28672 data: 4 3 2 3 3 3 4 23/30 Write block 149 len 36864 data: 4 3 3 3 2 2 2 3 4 24/30 Read block 155 len 32768 25/30 Read block 91 len 20480 26/30 Write block 467 len 24576 data: 2 3 3 2 2 2 27/30 Write block 370 len 24576 data: 4 4 2 3 3 3 28/30 Read block 139 len 20480 29/30 Read block 402 len 24576 30/30 Write block 230 len 24576 data: 2 2 5 4 4 4 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:11:59.606584984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 33 33 33 57 57 57 T T T 5 100-119 33 33 33 57 57 57 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 18 18 18 33 33 33 T T T 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 33 33 33 57 57 57 T T T 12 240-259 33 33 33 57 57 57 F F F 13 260-279 29 29 29 52 52 52 T T T 14 280-299 32 32 32 56 56 56 T T T 15 300-319 27 27 27 50 50 50 F F F 16 320-339 33 33 33 57 57 57 T T T 17 340-359 32 32 32 56 56 56 F F F 18 360-379 20 20 20 39 39 39 T T T 19 380-399 32 32 32 56 56 56 F F F 20 400-419 31 31 31 55 55 55 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 25 25 25 48 48 48 T T T 24 480-499 32 32 32 56 56 56 T T T 25 500-519 25 25 25 48 48 48 T T T 26 520-539 20 20 20 36 36 36 T T T 27 540-559 29 29 29 52 52 52 F F F 28 560-579 31 31 31 55 55 55 T T T 29 580-599 30 30 30 53 53 53 T T T Max gen: 33, Max flush: 57 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 48 48 48 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 25 25 25 48 48 48 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 54 54 54 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 33 33 33 57 57 57 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 27 27 27 50 50 50 F F F 16 320-339 34 34 34 58 58 58 F F F 17 340-359 32 32 32 56 56 56 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 31 31 31 55 55 55 F F F 21 420-439 18 18 18 33 33 33 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 29 29 29 52 52 52 F F F 28 560-579 34 34 34 58 58 58 F F F 29 580-599 34 34 34 58 58 58 F F F Max gen: 34, Max flush: 58 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.885625212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:03.928225672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.928588289Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.960226805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.960784633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.960820348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.960834056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.960848264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} {"msg":"Crucible 690f7f5a-0486-4fe1-86b0-01c72c3e0c6c has session id: 9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.961442628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Upstairs opts: Upstairs UUID: 690f7f5a-0486-4fe1-86b0-01c72c3e0c6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.961482712Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Crucible stats registered with UUID: 690f7f5a-0486-4fe1-86b0-01c72c3e0c6c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.961500919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:03.961522936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.961936846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.961965512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} {"msg":"690f7f5a-0486-4fe1-86b0-01c72c3e0c6c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962020014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962037751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962063637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962088054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96211575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962190489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962227063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96224764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962272537Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962297513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"ds_connection connected from Ok(127.0.0.1:54368)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96232101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"ds_connection connected from Ok(127.0.0.1:55340)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.962425275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"ds_connection connected from Ok(127.0.0.1:50949)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96245616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963017839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963049434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963120264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963749912Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963821102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.963855966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964397688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964470247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[0]R flush_numbers[0..12]: [48, 56, 56, 56, 58, 57, 48, 58, 54, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964513861Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[0]R generation[0..12]: [25, 32, 32, 32, 34, 33, 25, 34, 30, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964531498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964546346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[1]R flush_numbers[0..12]: [48, 56, 56, 56, 58, 57, 48, 58, 54, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964560214Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[1]R generation[0..12]: [25, 32, 32, 32, 34, 33, 25, 34, 30, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964573522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96458773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[2]R flush_numbers[0..12]: [48, 56, 56, 56, 58, 57, 48, 58, 54, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964601318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[2]R generation[0..12]: [25, 32, 32, 32, 34, 33, 25, 34, 30, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964618646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964632893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964645571Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96465854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964671088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964683526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"":"downstairs","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964712522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96472524Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"0","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964738318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"1","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-01-14T16:12:03.964757875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"client":"2","":"downstairs","session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964770953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964782782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"690f7f5a-0486-4fe1-86b0-01c72c3e0c6c is now active with session: 9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.96479536Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964807048Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326,"session_id":"9fbe79a9-d2dc-427c-9bf6-e3f1b8fec435"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:03.964819066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 4096 data: 3 02/30 Write block 311 len 16384 data: 3 3 3 2 03/30 Read block 285 len 28672 04/30 Write block 410 len 16384 data: 6 6 6 5 05/30 Write block 7 len 8192 data: 3 4 06/30 Write block 400 len 28672 data: 6 5 4 6 6 7 7 07/30 Flush 08/30 Write block 585 len 16384 data: 3 3 3 4 09/30 Write block 340 len 16384 data: 6 4 5 4 10/30 Read block 105 len 8192 11/30 Read block 376 len 36864 12/30 Read block 81 len 40960 13/30 Flush 14/30 Write block 422 len 4096 data: 3 15/30 Read block 203 len 16384 16/30 Read block 81 len 4096 17/30 Read block 433 len 28672 18/30 Read block 297 len 4096 19/30 Write block 544 len 24576 data: 3 3 4 3 4 4 20/30 Read block 551 len 16384 21/30 Read block 495 len 32768 22/30 Read block 458 len 4096 23/30 Write block 242 len 4096 data: 5 24/30 Write block 313 len 20480 data: 4 3 2 2 2 25/30 Read block 55 len 12288 26/30 Write block 442 len 24576 data: 3 3 2 2 2 4 27/30 Write block 128 len 16384 data: 2 2 3 3 28/30 Read block 451 len 20480 29/30 Write block 166 len 28672 data: 4 3 3 4 2 2 3 30/30 Write block 284 len 32768 data: 6 8 8 9 9 10 8 7 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:03.981750602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 25 25 25 48 48 48 T T T 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 54 54 54 T T T 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 33 33 33 57 57 57 T T T 13 260-279 36 36 36 59 59 59 F F F 14 280-299 34 34 34 58 58 58 T T T 15 300-319 36 36 36 59 59 59 T T T 16 320-339 34 34 34 58 58 58 F F F 17 340-359 36 36 36 60 60 60 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 36 36 36 59 59 59 F F F 21 420-439 18 18 18 33 33 33 T T T 22 440-459 20 20 20 39 39 39 T T T 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 29 29 29 52 52 52 T T T 28 560-579 34 34 34 58 58 58 F F F 29 580-599 36 36 36 60 60 60 F F F Max gen: 36, Max flush: 60 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 32 32 32 56 56 56 F F F 2 040-059 32 32 32 56 56 56 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 37 37 37 61 61 61 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 36 36 36 59 59 59 F F F 14 280-299 37 37 37 61 61 61 F F F 15 300-319 37 37 37 61 61 61 F F F 16 320-339 34 34 34 58 58 58 F F F 17 340-359 36 36 36 60 60 60 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 36 36 36 59 59 59 F F F 21 420-439 37 37 37 61 61 61 F F F 22 440-459 37 37 37 61 61 61 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 34 34 34 58 58 58 F F F 29 580-599 36 36 36 60 60 60 F F F Max gen: 37, Max flush: 61 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.26725167Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.311230259Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.311879545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.311923878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.311940996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.311958843Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} {"msg":"Crucible f64ffeb9-7758-491d-9e32-f8ae73156970 has session id: cb786ab0-8f87-4514-a462-1ac0b4a656dd","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.312468389Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Upstairs opts: Upstairs UUID: f64ffeb9-7758-491d-9e32-f8ae73156970, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.3125329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Crucible stats registered with UUID: f64ffeb9-7758-491d-9e32-f8ae73156970","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.312560126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:08.312580903Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313039426Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313063612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} {"msg":"f64ffeb9-7758-491d-9e32-f8ae73156970 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313125973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313173526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313187524Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313201632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.31321494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313228888Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313241567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313257774Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313271472Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47088)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313382166Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44342)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313418721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:64518)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.313445577Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.31410941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.314236442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.314725811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.314862951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.315118593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.31527892Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.315436287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.315720356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[0]R flush_numbers[0..12]: [59, 56, 56, 56, 58, 57, 61, 58, 61, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316275045Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[0]R generation[0..12]: [36, 32, 32, 32, 34, 33, 37, 34, 37, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316299091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316318049Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[1]R flush_numbers[0..12]: [59, 56, 56, 56, 58, 57, 61, 58, 61, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316332507Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[1]R generation[0..12]: [36, 32, 32, 32, 34, 33, 37, 34, 37, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316346425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316360433Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[2]R flush_numbers[0..12]: [59, 56, 56, 56, 58, 57, 61, 58, 61, 44, 50, 58]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316373721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[2]R generation[0..12]: [36, 32, 32, 32, 34, 33, 37, 34, 37, 22, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316386939Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316400317Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316412775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316425633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316438121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.3164506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"":"downstairs","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"No downstairs reconciliation requiredWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316469397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316481365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"0","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316494773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"1","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316510411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"client":"2","":"downstairs","session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316523779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316535777Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"f64ffeb9-7758-491d-9e32-f8ae73156970 is now active with session: cb786ab0-8f87-4514-a462-1ac0b4a656dd","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316548085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316559813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334,"session_id":"cb786ab0-8f87-4514-a462-1ac0b4a656dd"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:08.316571632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 165 len 40960 data: 6 5 4 4 5 3 3 4 4 4 02/30 Flush 03/30 Write block 39 len 8192 data: 9 8 04/30 Read block 434 len 4096 05/30 Write block 412 len 4096 data: 7 06/30 Read block 322 len 24576 07/30 Read block 485 len 40960 08/30 Write block 313 len 16384 data: 5 4 3 3 09/30 Read block 238 len 36864 10/30 Write block 588 len 12288 data: 5 3 3 11/30 Read block 296 len 8192 12/30 Write block 290 len 40960 data: 9 8 6 6 6 7 6 4 4 4 13/30 Read block 239 len 12288 14/30 Read block 25 len 12288 15/30 Write block 423 len 16384 data: 2 2 3 2 16/30 Read block 557 len 8192 17/30 Read block 493 len 20480 18/30 Read block 409 len 32768 19/30 Read block 551 len 16384 20/30 Write block 277 len 36864 data: 7 7 5 5 6 6 6 7 9 21/30 Write block 325 len 4096 data: 6 22/30 Read block 156 len 32768 23/30 Read block 168 len 32768 24/30 Write block 561 len 12288 data: 2 2 3 25/30 Write block 585 len 28672 data: 4 4 4 6 4 4 3 26/30 Read block 431 len 40960 27/30 Write block 213 len 40960 data: 3 3 4 6 6 5 4 5 5 4 28/30 Read block 265 len 4096 29/30 Read block 458 len 8192 30/30 Write block 514 len 16384 data: 5 5 5 6 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:08.336821435Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 32 32 32 56 56 56 T T T 2 040-059 32 32 32 56 56 56 T T T 3 060-079 32 32 32 56 56 56 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 27 27 27 50 50 50 T T T 11 220-239 34 34 34 58 58 58 T T T 12 240-259 37 37 37 61 61 61 F F F 13 260-279 36 36 36 59 59 59 T T T 14 280-299 37 37 37 61 61 61 T T T 15 300-319 37 37 37 61 61 61 T T T 16 320-339 34 34 34 58 58 58 T T T 17 340-359 36 36 36 60 60 60 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 36 36 36 59 59 59 T T T 21 420-439 37 37 37 61 61 61 T T T 22 440-459 37 37 37 61 61 61 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 34 34 34 58 58 58 T T T 26 520-539 34 34 34 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 34 34 34 58 58 58 T T T 29 580-599 36 36 36 60 60 60 T T T Max gen: 38, Max flush: 62 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 39 39 39 63 63 63 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 33 33 33 57 57 57 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 22 22 22 44 44 44 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 36 36 36 60 60 60 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 39 39 39 63 63 63 F F F 22 440-459 37 37 37 61 61 61 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 39, Max flush: 63 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.612304331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:12.655592381Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.65594671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.688411745Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.688982071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689019456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689035574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689054211Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} {"msg":"Crucible 522d6beb-c79b-4a2a-a11b-117b114f97d7 has session id: bcf985e6-dc57-4af7-9886-7d43f4579401","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689719684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Upstairs opts: Upstairs UUID: 522d6beb-c79b-4a2a-a11b-117b114f97d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689754869Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Crucible stats registered with UUID: 522d6beb-c79b-4a2a-a11b-117b114f97d7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.689771107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:12.689788234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690239368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690264435Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} {"msg":"522d6beb-c79b-4a2a-a11b-117b114f97d7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690324076Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690342053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.69036508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690378998Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690393026Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690406324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690420632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.69043379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690447448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690460926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"ds_connection connected from Ok(127.0.0.1:60724)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690474864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"ds_connection connected from Ok(127.0.0.1:61357)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.69057329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"ds_connection connected from Ok(127.0.0.1:42510)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.690619153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.691343828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.691381332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.691413618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.691965127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692051825Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692074632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692421031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692618692Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[0]R flush_numbers[0..12]: [59, 63, 63, 56, 58, 57, 61, 58, 62, 44, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692645798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[0]R generation[0..12]: [36, 39, 39, 32, 34, 33, 37, 34, 38, 22, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692661416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692676044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[1]R flush_numbers[0..12]: [59, 63, 63, 56, 58, 57, 61, 58, 62, 44, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692690152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[1]R generation[0..12]: [36, 39, 39, 32, 34, 33, 37, 34, 38, 22, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.69270378Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692717778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[2]R flush_numbers[0..12]: [59, 63, 63, 56, 58, 57, 61, 58, 62, 44, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692735035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[2]R generation[0..12]: [36, 39, 39, 32, 34, 33, 37, 34, 38, 22, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692748803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692762441Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.69277519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692788177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692800916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692813604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692833731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692854898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"0","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692875905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"1","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692889523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"client":"2","":"downstairs","session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692902371Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692914359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"522d6beb-c79b-4a2a-a11b-117b114f97d7 is now active with session: bcf985e6-dc57-4af7-9886-7d43f4579401","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692934806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692954863Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342,"session_id":"bcf985e6-dc57-4af7-9886-7d43f4579401"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:12.692970781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 40960 data: 3 3 5 5 5 5 5 5 4 4 02/30 Write block 99 len 24576 data: 7 7 7 5 5 5 03/30 Read block 285 len 36864 04/30 Flush 05/30 Write block 360 len 4096 data: 5 06/30 Read block 220 len 20480 07/30 Read block 0 len 24576 08/30 Write block 255 len 12288 data: 5 6 5 09/30 Read block 277 len 24576 10/30 Read block 49 len 28672 11/30 Read block 355 len 24576 12/30 Write block 334 len 16384 data: 5 7 7 8 13/30 Write block 190 len 40960 data: 5 5 5 5 4 4 4 3 2 2 14/30 Read block 269 len 20480 15/30 Write block 242 len 4096 data: 6 16/30 Flush 17/30 Write block 327 len 32768 data: 5 6 8 6 7 7 6 6 18/30 Read block 207 len 4096 19/30 Read block 22 len 12288 20/30 Write block 185 len 40960 data: 6 6 6 6 6 6 6 6 6 5 21/30 Read block 420 len 40960 22/30 Read block 511 len 12288 23/30 Flush 24/30 Write block 95 len 32768 data: 7 7 6 7 8 8 8 6 25/30 Write block 344 len 24576 data: 4 4 4 4 4 5 26/30 Read block 401 len 4096 27/30 Read block 577 len 12288 28/30 Write block 350 len 40960 data: 3 3 5 6 6 7 8 8 7 6 29/30 Read block 373 len 4096 30/30 Read block 106 len 8192 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:12.713198887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 39 39 39 63 63 63 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 40 40 40 64 64 64 T T T 5 100-119 40 40 40 64 64 64 T T T 6 120-139 37 37 37 61 61 61 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 40 40 40 65 65 65 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 40 40 40 66 66 66 F F F 17 340-359 36 36 36 60 60 60 T T T 18 360-379 40 40 40 65 65 65 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 39 39 39 63 63 63 F F F 22 440-459 37 37 37 61 61 61 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 40, Max flush: 66 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 39 39 39 63 63 63 F F F 3 060-079 32 32 32 56 56 56 F F F 4 080-099 41 41 41 67 67 67 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 40 40 40 65 65 65 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 39 39 39 63 63 63 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 40 40 40 66 66 66 F F F 17 340-359 41 41 41 67 67 67 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 39 39 39 63 63 63 F F F 22 440-459 37 37 37 61 61 61 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 41, Max flush: 67 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:16.968249216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.01364298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.014388261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.014441694Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.0144631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.014486147Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} {"msg":"Crucible 703320b8-2767-4b1f-a648-cbce06e6f86c has session id: d1d3972f-c6f9-43ad-a0df-56996f88d4e7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.01508634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Upstairs opts: Upstairs UUID: 703320b8-2767-4b1f-a648-cbce06e6f86c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.01515146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Crucible stats registered with UUID: 703320b8-2767-4b1f-a648-cbce06e6f86c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015171097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:17.015185325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015675554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.01569952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} {"msg":"703320b8-2767-4b1f-a648-cbce06e6f86c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015759842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015778849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015804755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015829841Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015861907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015887903Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015914409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015942435Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.015970201Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016003986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33455)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016029352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60642)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016055419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58455)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016265698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016766115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016794351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.016810129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.017464854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.017496059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.017512117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.01804146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018067836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[0]R flush_numbers[0..12]: [59, 63, 63, 56, 67, 67, 61, 58, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018182859Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[0]R generation[0..12]: [36, 39, 39, 32, 41, 41, 37, 34, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018204626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018223003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[1]R flush_numbers[0..12]: [59, 63, 63, 56, 67, 67, 61, 58, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018237141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[1]R generation[0..12]: [36, 39, 39, 32, 41, 41, 37, 34, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018250639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018264567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[2]R flush_numbers[0..12]: [59, 63, 63, 56, 67, 67, 61, 58, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018278025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[2]R generation[0..12]: [36, 39, 39, 32, 41, 41, 37, 34, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018291473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018305151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018317839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018330877Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-14T16:12:17.018374451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018391219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"":"downstairs","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018404277Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018416055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"0","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018429203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"1","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018441841Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"client":"2","":"downstairs","session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018457329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018469547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"703320b8-2767-4b1f-a648-cbce06e6f86c is now active with session: d1d3972f-c6f9-43ad-a0df-56996f88d4e7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018482015Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018493714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350,"session_id":"d1d3972f-c6f9-43ad-a0df-56996f88d4e7"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:17.018505632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 145 len 16384 data: 4 3 4 4 03/30 Read block 465 len 8192 04/30 Write block 255 len 8192 data: 6 7 05/30 Read block 145 len 24576 06/30 Write block 79 len 20480 data: 4 4 3 3 4 07/30 Write block 267 len 40960 data: 4 4 5 4 4 4 4 6 6 6 08/30 Read block 181 len 16384 09/30 Read block 54 len 28672 10/30 Write block 51 len 40960 data: 5 6 5 5 5 5 5 4 3 3 11/30 Read block 137 len 32768 12/30 Write block 443 len 20480 data: 4 3 3 3 5 13/30 Read block 172 len 36864 14/30 Read block 326 len 8192 15/30 Read block 450 len 28672 16/30 Read block 6 len 8192 17/30 Read block 200 len 28672 18/30 Read block 191 len 12288 19/30 Read block 193 len 12288 20/30 Write block 532 len 20480 data: 3 3 3 4 4 21/30 Write block 147 len 20480 data: 5 5 5 4 4 22/30 Write block 329 len 24576 data: 9 7 8 8 7 7 23/30 Flush 24/30 Write block 293 len 16384 data: 7 7 8 7 25/30 Read block 212 len 8192 26/30 Write block 61 len 20480 data: 3 6 6 6 7 27/30 Read block 397 len 4096 28/30 Read block 207 len 16384 29/30 Read block 305 len 32768 30/30 Write block 264 len 28672 data: 4 3 4 5 5 6 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:17.038012553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 42 42 42 69 69 69 T T T 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 42 42 42 69 69 69 T T T 14 280-299 39 39 39 63 63 63 T T T 15 300-319 39 39 39 63 63 63 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 41 41 41 67 67 67 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 39 39 39 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 42 42 42 69 69 69 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 42, Max flush: 69 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 43 43 43 70 70 70 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 37 37 37 61 61 61 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 39 39 39 63 63 63 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 43 43 43 70 70 70 F F F 14 280-299 43 43 43 70 70 70 F F F 15 300-319 39 39 39 63 63 63 F F F 16 320-339 42 42 42 69 69 69 F F F 17 340-359 41 41 41 67 67 67 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 32 32 32 56 56 56 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 39 39 39 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 42 42 42 69 69 69 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 43, Max flush: 70 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.263146178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.307638892Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308278089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308330731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308345769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308360097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} {"msg":"Crucible 372ce717-7453-40d1-9093-0f0cafbacbe3 has session id: d14a4a6c-94fd-4347-8b01-353c9d69b10f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308910837Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Upstairs opts: Upstairs UUID: 372ce717-7453-40d1-9093-0f0cafbacbe3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308944662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Crucible stats registered with UUID: 372ce717-7453-40d1-9093-0f0cafbacbe3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.308967259Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:21.308981517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309398636Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309422623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} {"msg":"372ce717-7453-40d1-9093-0f0cafbacbe3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309474245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309500381Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309523748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309537976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309552124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309568671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309583719Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309605836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309630012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309653879Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56974)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309691463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53458)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309722109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45697)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.309912051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.310288886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.310451253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.310574175Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.310957809Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.310986155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311239448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.31149847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311612034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[0]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 61, 69, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311720018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[0]R generation[0..12]: [36, 39, 42, 43, 42, 41, 37, 42, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311740925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311755543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[1]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 61, 69, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311769371Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[1]R generation[0..12]: [36, 39, 42, 43, 42, 41, 37, 42, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311782839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311796957Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[2]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 61, 69, 62, 66, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311810255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[2]R generation[0..12]: [36, 39, 42, 43, 42, 41, 37, 42, 38, 40, 39, 39]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311823333Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311836861Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311849309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311862057Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311874486Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-14T16:12:21.311896232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"":"downstairs","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.31190923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311920969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"0","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311933887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"1","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311946585Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"client":"2","":"downstairs","session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311959273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311971192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"372ce717-7453-40d1-9093-0f0cafbacbe3 is now active with session: d14a4a6c-94fd-4347-8b01-353c9d69b10f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.31198359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.311997578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358,"session_id":"d14a4a6c-94fd-4347-8b01-353c9d69b10f"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:21.312019745Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 292 len 32768 data: 7 8 8 9 8 5 5 5 02/30 Read block 94 len 20480 03/30 Write block 337 len 28672 data: 9 9 8 7 5 6 5 04/30 Read block 306 len 36864 05/30 Write block 430 len 4096 data: 4 06/30 Read block 553 len 36864 07/30 Write block 304 len 4096 data: 4 08/30 Read block 364 len 32768 09/30 Read block 142 len 12288 10/30 Write block 180 len 20480 data: 4 3 4 4 6 11/30 Write block 168 len 16384 data: 5 6 4 4 12/30 Write block 468 len 4096 data: 4 13/30 Read block 63 len 36864 14/30 Read block 53 len 40960 15/30 Write block 486 len 16384 data: 4 4 6 6 16/30 Write block 126 len 36864 data: 3 2 3 3 4 4 3 3 3 17/30 Read block 0 len 32768 18/30 Read block 344 len 8192 19/30 Write block 387 len 40960 data: 5 6 6 5 3 4 4 4 4 4 20/30 Flush 21/30 Write block 466 len 4096 data: 3 22/30 Read block 16 len 20480 23/30 Read block 371 len 28672 24/30 Read block 336 len 24576 25/30 Write block 208 len 36864 data: 4 4 4 4 5 4 4 5 7 26/30 Write block 374 len 8192 data: 4 4 27/30 Write block 395 len 16384 data: 5 5 7 7 28/30 Read block 137 len 12288 29/30 Read block 180 len 28672 30/30 Write block 234 len 12288 data: 5 5 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:21.331660436Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 43 43 43 70 70 70 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 44 44 44 71 71 71 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 44 44 44 71 71 71 F F F 10 200-219 39 39 39 63 63 63 T T T 11 220-239 39 39 39 63 63 63 T T T 12 240-259 42 42 42 69 69 69 F F F 13 260-279 43 43 43 70 70 70 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 44 44 44 71 71 71 F F F 16 320-339 44 44 44 71 71 71 F F F 17 340-359 44 44 44 71 71 71 F F F 18 360-379 40 40 40 65 65 65 T T T 19 380-399 44 44 44 71 71 71 T T T 20 400-419 39 39 39 63 63 63 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 71 71 71 T T T 24 480-499 44 44 44 71 71 71 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 42 42 42 69 69 69 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 44, Max flush: 71 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 43 43 43 70 70 70 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 44 44 44 71 71 71 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 44 44 44 71 71 71 F F F 10 200-219 45 45 45 72 72 72 F F F 11 220-239 45 45 45 72 72 72 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 43 43 43 70 70 70 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 44 44 44 71 71 71 F F F 16 320-339 44 44 44 71 71 71 F F F 17 340-359 44 44 44 71 71 71 F F F 18 360-379 45 45 45 72 72 72 F F F 19 380-399 45 45 45 72 72 72 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 45 45 45 72 72 72 F F F 24 480-499 44 44 44 71 71 71 F F F 25 500-519 39 39 39 63 63 63 F F F 26 520-539 42 42 42 69 69 69 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 39 39 39 63 63 63 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 45, Max flush: 72 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.567285123Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.613576086Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614214763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614257657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614272964Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614287452Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} {"msg":"Crucible 64d8ee3f-a753-480d-b117-d170746d08e8 has session id: 689f16d7-f6f9-49f8-b779-77bb075f4ac5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614892314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Upstairs opts: Upstairs UUID: 64d8ee3f-a753-480d-b117-d170746d08e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614926269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Crucible stats registered with UUID: 64d8ee3f-a753-480d-b117-d170746d08e8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.614941767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:25.614954465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615318972Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615351687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} {"msg":"64d8ee3f-a753-480d-b117-d170746d08e8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615408859Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615439265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615477799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615505055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615533611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615560397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615589693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.61560961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615635646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615658493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60149)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.61568127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63944)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615742721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62366)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.615812071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.6163639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.616401725Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.616466095Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.616998498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617026554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617140797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617513513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.61759927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[0]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 71, 69, 71, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617701765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[0]R generation[0..12]: [36, 39, 42, 43, 42, 41, 44, 42, 44, 44, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617720993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617735771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[1]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 71, 69, 71, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617749129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[1]R generation[0..12]: [36, 39, 42, 43, 42, 41, 44, 42, 44, 44, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617762437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617776125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[2]R flush_numbers[0..12]: [59, 63, 69, 70, 69, 67, 71, 69, 71, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617789593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[2]R generation[0..12]: [36, 39, 42, 43, 42, 41, 44, 42, 44, 44, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617802721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617819688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617832926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617846354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617859022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.61787771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"":"downstairs","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617891048Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617902736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"0","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617915744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"1","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617928603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"client":"2","":"downstairs","session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617941541Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617953439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"64d8ee3f-a753-480d-b117-d170746d08e8 is now active with session: 689f16d7-f6f9-49f8-b779-77bb075f4ac5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617965797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617977315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366,"session_id":"689f16d7-f6f9-49f8-b779-77bb075f4ac5"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:25.617989094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 454 len 32768 data: 6 6 4 5 4 4 4 3 02/30 Write block 266 len 4096 data: 5 03/30 Write block 317 len 32768 data: 3 2 3 2 2 3 4 6 04/30 Read block 542 len 16384 05/30 Write block 129 len 20480 data: 4 5 5 4 4 06/30 Read block 240 len 8192 07/30 Read block 514 len 28672 08/30 Write block 565 len 24576 data: 3 3 3 3 3 3 09/30 Write block 63 len 40960 data: 7 7 8 4 2 2 3 3 2 2 10/30 Flush 11/30 Write block 518 len 12288 data: 7 7 5 12/30 Write block 519 len 4096 data: 8 13/30 Read block 380 len 40960 14/30 Read block 511 len 28672 15/30 Flush 16/30 Write block 457 len 16384 data: 6 5 5 5 17/30 Read block 60 len 40960 18/30 Flush 19/30 Flush 20/30 Write block 28 len 4096 data: 3 21/30 Read block 117 len 28672 22/30 Write block 73 len 28672 data: 3 3 5 5 5 4 5 23/30 Read block 379 len 8192 24/30 Read block 442 len 32768 25/30 Write block 228 len 8192 data: 2 2 26/30 Read block 496 len 20480 27/30 Write block 18 len 12288 data: 3 3 3 28/30 Read block 592 len 24576 29/30 Read block 498 len 32768 30/30 Write block 425 len 24576 data: 4 3 2 3 3 5 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:25.636640029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 59 59 59 T T T 1 020-039 39 39 39 63 63 63 T T T 2 040-059 42 42 42 69 69 69 F F F 3 060-079 46 46 46 73 73 73 T T T 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 46 46 46 73 73 73 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 44 44 44 71 71 71 F F F 10 200-219 45 45 45 72 72 72 F F F 11 220-239 45 45 45 72 72 72 T T T 12 240-259 42 42 42 69 69 69 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 46 46 46 73 73 73 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 44 44 44 71 71 71 F F F 18 360-379 45 45 45 72 72 72 F F F 19 380-399 45 45 45 72 72 72 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 44 44 44 71 71 71 T T T 22 440-459 46 46 46 75 75 75 F F F 23 460-479 46 46 46 75 75 75 F F F 24 480-499 44 44 44 71 71 71 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 46 46 46 74 74 74 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 46, Max flush: 75 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 47 47 47 76 76 76 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 46 46 46 73 73 73 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 44 44 44 71 71 71 F F F 10 200-219 45 45 45 72 72 72 F F F 11 220-239 47 47 47 76 76 76 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 46 46 46 73 73 73 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 44 44 44 71 71 71 F F F 18 360-379 45 45 45 72 72 72 F F F 19 380-399 45 45 45 72 72 72 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 47 47 47 76 76 76 F F F 22 440-459 46 46 46 75 75 75 F F F 23 460-479 46 46 46 75 75 75 F F F 24 480-499 44 44 44 71 71 71 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 46 46 46 74 74 74 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 39 39 39 63 63 63 F F F Max gen: 47, Max flush: 76 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.871133831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:29.913649083Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.914055064Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.945878513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.946460328Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.946495213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.946509171Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.946523759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} {"msg":"Crucible d61d7df9-310d-4455-94a0-1e6076c81481 has session id: 91913d60-efe6-46ec-9a70-6197fb34c7c4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947105334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Upstairs opts: Upstairs UUID: d61d7df9-310d-4455-94a0-1e6076c81481, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947146248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Crucible stats registered with UUID: d61d7df9-310d-4455-94a0-1e6076c81481","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947170465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:29.947190651Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947626288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947660653Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} {"msg":"d61d7df9-310d-4455-94a0-1e6076c81481 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947713795Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947740491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947771507Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947797043Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947825729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947852555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947880351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947909587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.947943232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948133884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34610)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948166669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33242)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948197025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54506)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948225261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948707271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948738376Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.948908792Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.949393731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.949422097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.949457621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.94988224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.949992304Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[0]R flush_numbers[0..12]: [76, 76, 69, 76, 69, 67, 73, 69, 71, 71, 72, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950060444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[0]R generation[0..12]: [47, 47, 42, 47, 42, 41, 46, 42, 44, 44, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950078741Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950097279Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[1]R flush_numbers[0..12]: [76, 76, 69, 76, 69, 67, 73, 69, 71, 71, 72, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950111057Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[1]R generation[0..12]: [47, 47, 42, 47, 42, 41, 46, 42, 44, 44, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950124375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950138073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[2]R flush_numbers[0..12]: [76, 76, 69, 76, 69, 67, 73, 69, 71, 71, 72, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950151241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[2]R generation[0..12]: [47, 47, 42, 47, 42, 41, 46, 42, 44, 44, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950164449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950177987Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950190735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950203493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.95022223Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950235178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"":"downstairs","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950247966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950259845Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"0","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950272833Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"1","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950288021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"client":"2","":"downstairs","session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950301169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950313467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"d61d7df9-310d-4455-94a0-1e6076c81481 is now active with session: 91913d60-efe6-46ec-9a70-6197fb34c7c4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950326335Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950338093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374,"session_id":"91913d60-efe6-46ec-9a70-6197fb34c7c4"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:29.950350092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 197 len 28672 data: 4 3 3 2 2 2 2 02/30 Read block 262 len 32768 03/30 Flush 04/30 Write block 555 len 32768 data: 3 4 2 2 2 2 3 3 05/30 Write block 46 len 28672 data: 5 6 5 5 5 6 7 06/30 Write block 132 len 20480 data: 5 5 4 3 3 07/30 Write block 342 len 40960 data: 7 6 5 5 5 5 5 6 4 4 08/30 Write block 214 len 8192 data: 5 6 09/30 Read block 377 len 32768 10/30 Flush 11/30 Write block 344 len 12288 data: 6 6 6 12/30 Write block 306 len 20480 data: 3 3 3 3 3 13/30 Read block 37 len 16384 14/30 Read block 347 len 36864 15/30 Read block 537 len 8192 16/30 Read block 121 len 24576 17/30 Write block 584 len 12288 data: 3 5 5 18/30 Read block 253 len 40960 19/30 Write block 360 len 12288 data: 6 6 7 20/30 Flush 21/30 Write block 111 len 8192 data: 4 4 22/30 Read block 81 len 32768 23/30 Read block 143 len 40960 24/30 Read block 392 len 40960 25/30 Read block 520 len 8192 26/30 Write block 573 len 28672 data: 4 5 5 4 5 4 4 27/30 Write block 427 len 36864 data: 3 4 4 6 4 3 2 2 5 28/30 Write block 107 len 12288 data: 4 4 4 29/30 Read block 430 len 24576 30/30 Write block 548 len 24576 data: 5 5 3 3 3 3 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:29.96849949Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 47 47 47 76 76 76 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 41 41 41 67 67 67 T T T 6 120-139 48 48 48 78 78 78 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 47 47 47 76 76 76 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 48 48 48 79 79 79 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 48 48 48 79 79 79 F F F 18 360-379 48 48 48 79 79 79 F F F 19 380-399 45 45 45 72 72 72 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 47 47 47 76 76 76 T T T 22 440-459 46 46 46 75 75 75 F F F 23 460-479 46 46 46 75 75 75 F F F 24 480-499 44 44 44 71 71 71 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 46 46 46 74 74 74 F F F 27 540-559 48 48 48 78 78 78 T T T 28 560-579 48 48 48 78 78 78 T T T 29 580-599 48 48 48 79 79 79 F F F Max gen: 48, Max flush: 79 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 47 47 47 76 76 76 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 42 42 42 69 69 69 F F F 5 100-119 49 49 49 80 80 80 F F F 6 120-139 48 48 48 78 78 78 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 47 47 47 76 76 76 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 44 44 44 71 71 71 F F F 15 300-319 48 48 48 79 79 79 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 48 48 48 79 79 79 F F F 18 360-379 48 48 48 79 79 79 F F F 19 380-399 45 45 45 72 72 72 F F F 20 400-419 39 39 39 63 63 63 F F F 21 420-439 49 49 49 80 80 80 F F F 22 440-459 46 46 46 75 75 75 F F F 23 460-479 46 46 46 75 75 75 F F F 24 480-499 44 44 44 71 71 71 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 46 46 46 74 74 74 F F F 27 540-559 49 49 49 80 80 80 F F F 28 560-579 49 49 49 80 80 80 F F F 29 580-599 48 48 48 79 79 79 F F F Max gen: 49, Max flush: 80 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.234444155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.279655525Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.280589369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.280631703Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.280646431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.280660838Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} {"msg":"Crucible f3c7b802-1bd7-4e60-ba3e-33b10dd295b5 has session id: cd47cc11-5d44-4ea1-91dd-9b1fc12cab54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281311614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Upstairs opts: Upstairs UUID: f3c7b802-1bd7-4e60-ba3e-33b10dd295b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281355367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Crucible stats registered with UUID: f3c7b802-1bd7-4e60-ba3e-33b10dd295b5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281379054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:34.281399031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281868763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281895789Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} {"msg":"f3c7b802-1bd7-4e60-ba3e-33b10dd295b5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281950191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.281975887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282000913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282016121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282034449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282049296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282075392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282100789Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282126025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282153331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"ds_connection connected from Ok(127.0.0.1:65107)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282177238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"ds_connection connected from Ok(127.0.0.1:42033)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282209393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"ds_connection connected from Ok(127.0.0.1:54556)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282325806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282945836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.282977031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.283610169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.283704295Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.283916944Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.284174047Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.284257305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.284511468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[0]R flush_numbers[0..12]: [76, 76, 78, 76, 69, 80, 78, 69, 71, 77, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285016624Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[0]R generation[0..12]: [47, 47, 48, 47, 42, 49, 48, 42, 44, 48, 48, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285037781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285052129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[1]R flush_numbers[0..12]: [76, 76, 78, 76, 69, 80, 78, 69, 71, 77, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285065567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[1]R generation[0..12]: [47, 47, 48, 47, 42, 49, 48, 42, 44, 48, 48, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285078686Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285092384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[2]R flush_numbers[0..12]: [76, 76, 78, 76, 69, 80, 78, 69, 71, 77, 78, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285105722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[2]R generation[0..12]: [47, 47, 48, 47, 42, 49, 48, 42, 44, 48, 48, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285128748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285154774Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285177881Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:12:34.285191919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285214356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285227324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"":"downstairs","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285240312Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.28525185Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"0","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285265028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"1","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285277837Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"client":"2","":"downstairs","session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285290545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285302493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"f3c7b802-1bd7-4e60-ba3e-33b10dd295b5 is now active with session: cd47cc11-5d44-4ea1-91dd-9b1fc12cab54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285315011Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285326439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382,"session_id":"cd47cc11-5d44-4ea1-91dd-9b1fc12cab54"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:34.285338208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 40960 data: 4 5 5 5 6 6 8 8 7 7 02/30 Write block 503 len 40960 data: 6 7 7 7 8 8 7 6 5 4 03/30 Write block 226 len 28672 data: 4 4 3 3 3 3 6 04/30 Write block 576 len 28672 data: 5 6 5 5 4 4 3 05/30 Write block 221 len 32768 data: 6 5 5 3 3 5 5 4 06/30 Flush 07/30 Write block 530 len 8192 data: 3 3 08/30 Write block 506 len 16384 data: 8 9 9 8 09/30 Flush 10/30 Write block 259 len 4096 data: 4 11/30 Write block 566 len 28672 data: 4 4 4 4 4 3 4 12/30 Write block 425 len 28672 data: 5 4 4 5 5 7 5 13/30 Read block 180 len 40960 14/30 Write block 202 len 40960 data: 3 3 2 3 3 3 5 5 5 5 15/30 Read block 579 len 32768 16/30 Read block 248 len 28672 17/30 Write block 381 len 12288 data: 4 5 5 18/30 Write block 22 len 28672 data: 2 3 3 3 3 3 4 19/30 Write block 231 len 20480 data: 4 7 5 6 6 20/30 Read block 141 len 28672 21/30 Read block 96 len 36864 22/30 Read block 82 len 12288 23/30 Write block 284 len 20480 data: 8 10 9 10 10 24/30 Write block 449 len 8192 data: 4 4 25/30 Read block 41 len 16384 26/30 Write block 377 len 20480 data: 4 3 3 2 5 27/30 Write block 536 len 16384 data: 5 4 5 4 28/30 Write block 481 len 28672 data: 5 5 4 4 4 5 5 29/30 Read block 160 len 28672 30/30 Write block 96 len 24576 data: 8 7 8 9 9 9 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:34.304119454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 47 47 47 76 76 76 T T T 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 42 42 42 69 69 69 T T T 5 100-119 49 49 49 80 80 80 T T T 6 120-139 48 48 48 78 78 78 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 48 48 48 78 78 78 T T T 11 220-239 50 50 50 81 81 81 T T T 12 240-259 42 42 42 69 69 69 T T T 13 260-279 46 46 46 73 73 73 F F F 14 280-299 44 44 44 71 71 71 T T T 15 300-319 48 48 48 79 79 79 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 48 48 48 79 79 79 F F F 18 360-379 48 48 48 79 79 79 T T T 19 380-399 50 50 50 81 81 81 T T T 20 400-419 50 50 50 81 81 81 F F F 21 420-439 49 49 49 80 80 80 T T T 22 440-459 46 46 46 75 75 75 T T T 23 460-479 46 46 46 75 75 75 F F F 24 480-499 44 44 44 71 71 71 T T T 25 500-519 50 50 50 82 82 82 F F F 26 520-539 50 50 50 82 82 82 T T T 27 540-559 49 49 49 80 80 80 F F F 28 560-579 50 50 50 81 81 81 T T T 29 580-599 50 50 50 81 81 81 F F F Max gen: 50, Max flush: 82 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 51 51 51 83 83 83 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 51 51 51 83 83 83 F F F 6 120-139 48 48 48 78 78 78 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 51 51 51 83 83 83 F F F 11 220-239 51 51 51 83 83 83 F F F 12 240-259 51 51 51 83 83 83 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 51 51 51 83 83 83 F F F 15 300-319 48 48 48 79 79 79 F F F 16 320-339 46 46 46 73 73 73 F F F 17 340-359 48 48 48 79 79 79 F F F 18 360-379 51 51 51 83 83 83 F F F 19 380-399 51 51 51 83 83 83 F F F 20 400-419 50 50 50 81 81 81 F F F 21 420-439 51 51 51 83 83 83 F F F 22 440-459 51 51 51 83 83 83 F F F 23 460-479 46 46 46 75 75 75 F F F 24 480-499 51 51 51 83 83 83 F F F 25 500-519 50 50 50 82 82 82 F F F 26 520-539 51 51 51 83 83 83 F F F 27 540-559 49 49 49 80 80 80 F F F 28 560-579 51 51 51 83 83 83 F F F 29 580-599 50 50 50 81 81 81 F F F Max gen: 51, Max flush: 83 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.5868155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.631464362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632079363Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632121956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632137274Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632151512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} {"msg":"Crucible 580ec4b4-e56a-4a8e-87c1-37c0d0d4014c has session id: 352b668f-fbc5-4f3e-afa2-a8ac426f262e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632793179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Upstairs opts: Upstairs UUID: 580ec4b4-e56a-4a8e-87c1-37c0d0d4014c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.632833563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Crucible stats registered with UUID: 580ec4b4-e56a-4a8e-87c1-37c0d0d4014c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.63285449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:38.632868478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633247282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633272069Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} {"msg":"580ec4b4-e56a-4a8e-87c1-37c0d0d4014c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633326481Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633343239Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633365035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633378403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633392341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.63340529Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633420267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633441484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633465921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633488677Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48507)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633513054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36783)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633559077Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51899)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.633589143Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.634291041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.634320356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.634341693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.634973001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635016675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635033313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"1","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635551827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635584752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[0]R flush_numbers[0..12]: [76, 83, 78, 76, 83, 83, 78, 69, 71, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.63559788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[0]R generation[0..12]: [47, 51, 48, 47, 51, 51, 48, 42, 44, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635612118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635626006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[1]R flush_numbers[0..12]: [76, 83, 78, 76, 83, 83, 78, 69, 71, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635639544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[1]R generation[0..12]: [47, 51, 48, 47, 51, 51, 48, 42, 44, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635656122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.63567033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[2]R flush_numbers[0..12]: [76, 83, 78, 76, 83, 83, 78, 69, 71, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635683978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[2]R generation[0..12]: [47, 51, 48, 47, 51, 51, 48, 42, 44, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635697276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635710994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635723762Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.6357366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635749058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635761517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"":"downstairs","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635774115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635785703Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"0","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635798751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"1","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635819558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"client":"2","":"downstairs","session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635832276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635844115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"580ec4b4-e56a-4a8e-87c1-37c0d0d4014c is now active with session: 352b668f-fbc5-4f3e-afa2-a8ac426f262e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635860162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.63587282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390,"session_id":"352b668f-fbc5-4f3e-afa2-a8ac426f262e"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:38.635884939Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 32768 data: 5 6 6 5 6 7 6 5 02/30 Write block 30 len 20480 data: 3 4 7 5 5 03/30 Write block 543 len 28672 data: 3 4 4 5 4 6 6 04/30 Write block 294 len 36864 data: 9 10 9 6 6 6 4 4 3 05/30 Write block 149 len 20480 data: 6 5 5 4 3 06/30 Read block 275 len 40960 07/30 Read block 256 len 32768 08/30 Read block 31 len 8192 09/30 Write block 551 len 4096 data: 4 10/30 Read block 116 len 36864 11/30 Write block 390 len 16384 data: 6 5 6 6 12/30 Read block 414 len 20480 13/30 Flush 14/30 Write block 338 len 12288 data: 10 9 8 15/30 Read block 560 len 12288 16/30 Write block 557 len 28672 data: 3 3 3 3 4 4 4 17/30 Read block 186 len 20480 18/30 Flush 19/30 Write block 377 len 28672 data: 5 4 4 3 6 6 6 20/30 Write block 503 len 24576 data: 7 8 8 9 10 10 21/30 Read block 35 len 12288 22/30 Read block 323 len 16384 23/30 Read block 389 len 8192 24/30 Read block 516 len 4096 25/30 Read block 414 len 8192 26/30 Write block 472 len 28672 data: 3 2 2 4 4 4 4 27/30 Write block 117 len 20480 data: 3 2 2 4 4 28/30 Read block 264 len 16384 29/30 Write block 208 len 28672 data: 6 6 6 6 6 5 6 30/30 Read block 355 len 40960 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:38.65470242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 51 51 51 83 83 83 T T T 6 120-139 48 48 48 78 78 78 T T T 7 140-159 52 52 52 84 84 84 F F F 8 160-179 52 52 52 84 84 84 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 51 51 51 83 83 83 T T T 11 220-239 51 51 51 83 83 83 F F F 12 240-259 51 51 51 83 83 83 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 52 52 52 84 84 84 F F F 16 320-339 52 52 52 85 85 85 F F F 17 340-359 52 52 52 85 85 85 F F F 18 360-379 51 51 51 83 83 83 T T T 19 380-399 52 52 52 84 84 84 T T T 20 400-419 50 50 50 81 81 81 F F F 21 420-439 51 51 51 83 83 83 F F F 22 440-459 51 51 51 83 83 83 F F F 23 460-479 46 46 46 75 75 75 T T T 24 480-499 51 51 51 83 83 83 F F F 25 500-519 50 50 50 82 82 82 T T T 26 520-539 51 51 51 83 83 83 F F F 27 540-559 52 52 52 85 85 85 F F F 28 560-579 52 52 52 85 85 85 F F F 29 580-599 50 50 50 81 81 81 F F F Max gen: 52, Max flush: 85 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 53 53 53 86 86 86 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 52 52 52 84 84 84 F F F 8 160-179 52 52 52 84 84 84 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 53 53 53 86 86 86 F F F 11 220-239 51 51 51 83 83 83 F F F 12 240-259 51 51 51 83 83 83 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 52 52 52 84 84 84 F F F 16 320-339 52 52 52 85 85 85 F F F 17 340-359 52 52 52 85 85 85 F F F 18 360-379 53 53 53 86 86 86 F F F 19 380-399 53 53 53 86 86 86 F F F 20 400-419 50 50 50 81 81 81 F F F 21 420-439 51 51 51 83 83 83 F F F 22 440-459 51 51 51 83 83 83 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 51 51 51 83 83 83 F F F 25 500-519 53 53 53 86 86 86 F F F 26 520-539 51 51 51 83 83 83 F F F 27 540-559 52 52 52 85 85 85 F F F 28 560-579 52 52 52 85 85 85 F F F 29 580-599 50 50 50 81 81 81 F F F Max gen: 53, Max flush: 86 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.898895079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:42.941458355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.941829221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.973389668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.973944717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.973978392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97399209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.974015716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} {"msg":"Crucible e6709481-a7e1-499a-9449-700b7ebb2420 has session id: 443b3b28-c8b6-4961-8ec0-66df1607fb8b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.974629257Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Upstairs opts: Upstairs UUID: e6709481-a7e1-499a-9449-700b7ebb2420, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.974662622Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Crucible stats registered with UUID: e6709481-a7e1-499a-9449-700b7ebb2420","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97467763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:42.974690098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975147532Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975171258Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} {"msg":"e6709481-a7e1-499a-9449-700b7ebb2420 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975238368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975255656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975277583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975291151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975308968Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975322546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975344603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975369359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975396236Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975421722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35492)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975448218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48013)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975474084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46189)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.975697462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97619231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.976222245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.976238103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.976849794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97687928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.976930772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97749144Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977518177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[0]R flush_numbers[0..12]: [76, 84, 78, 76, 83, 86, 86, 84, 84, 77, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977532885Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[0]R generation[0..12]: [47, 52, 48, 47, 51, 53, 53, 52, 52, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977547682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97756211Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[1]R flush_numbers[0..12]: [76, 84, 78, 76, 83, 86, 86, 84, 84, 77, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977575928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[1]R generation[0..12]: [47, 52, 48, 47, 51, 53, 53, 52, 52, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977589436Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977603204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[2]R flush_numbers[0..12]: [76, 84, 78, 76, 83, 86, 86, 84, 84, 77, 86, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977616572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[2]R generation[0..12]: [47, 52, 48, 47, 51, 53, 53, 52, 52, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97762971Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977643138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977655597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977668515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977684922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.97769793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"":"downstairs","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977710899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"Wait for a query_work_queue command to finish before sending IO 443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977728526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"0","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977741574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"1","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977754282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"client":"2","":"downstairs","session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977766951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977778889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"e6709481-a7e1-499a-9449-700b7ebb2420 is now active with session: 443b3b28-c8b6-4961-8ec0-66df1607fb8b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977791357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977802835Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398,"session_id":"443b3b28-c8b6-4961-8ec0-66df1607fb8b"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:42.977814593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 36864 data: 7 7 7 6 7 7 6 7 7 02/30 Read block 529 len 24576 03/30 Write block 582 len 12288 data: 4 2 4 04/30 Read block 15 len 12288 05/30 Read block 51 len 24576 06/30 Read block 298 len 24576 07/30 Read block 530 len 24576 08/30 Read block 320 len 4096 09/30 Read block 286 len 20480 10/30 Write block 527 len 24576 data: 3 3 3 4 4 4 11/30 Read block 311 len 16384 12/30 Read block 345 len 28672 13/30 Write block 335 len 4096 data: 8 14/30 Read block 62 len 28672 15/30 Write block 65 len 32768 data: 9 5 3 3 4 4 3 3 16/30 Read block 545 len 4096 17/30 Write block 392 len 8192 data: 8 8 18/30 Write block 145 len 20480 data: 5 4 6 6 7 19/30 Read block 540 len 32768 20/30 Read block 41 len 12288 21/30 Write block 567 len 28672 data: 5 5 5 5 4 5 5 22/30 Write block 214 len 4096 data: 7 23/30 Write block 405 len 24576 data: 8 8 6 6 6 7 24/30 Write block 420 len 4096 data: 3 25/30 Read block 333 len 4096 26/30 Read block 448 len 20480 27/30 Read block 33 len 28672 28/30 Write block 412 len 32768 data: 8 6 6 5 5 5 4 4 29/30 Write block 342 len 40960 data: 8 7 7 7 7 6 6 7 5 5 30/30 Write block 65 len 40960 data: 10 6 4 4 5 5 4 4 4 4 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 29 28 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:42.996684247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 47 47 47 76 76 76 T T T 4 080-099 51 51 51 83 83 83 F F F 5 100-119 53 53 53 86 86 86 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 52 52 52 84 84 84 T T T 8 160-179 52 52 52 84 84 84 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 53 53 53 86 86 86 T T T 11 220-239 51 51 51 83 83 83 F F F 12 240-259 51 51 51 83 83 83 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 52 52 52 84 84 84 F F F 16 320-339 52 52 52 85 85 85 T T T 17 340-359 52 52 52 85 85 85 T T T 18 360-379 53 53 53 86 86 86 F F F 19 380-399 53 53 53 86 86 86 T T T 20 400-419 50 50 50 81 81 81 T T T 21 420-439 51 51 51 83 83 83 T T T 22 440-459 51 51 51 83 83 83 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 51 51 51 83 83 83 F F F 25 500-519 53 53 53 86 86 86 F F F 26 520-539 51 51 51 83 83 83 T T T 27 540-559 52 52 52 85 85 85 F F F 28 560-579 52 52 52 85 85 85 T T T 29 580-599 50 50 50 81 81 81 T T T Max gen: 53, Max flush: 86 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 48 48 48 78 78 78 F F F 3 060-079 54 54 54 87 87 87 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 53 53 53 86 86 86 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 52 52 52 84 84 84 F F F 9 180-199 48 48 48 77 77 77 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 51 51 51 83 83 83 F F F 12 240-259 51 51 51 83 83 83 F F F 13 260-279 46 46 46 73 73 73 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 52 52 52 84 84 84 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 53 53 53 86 86 86 F F F 19 380-399 54 54 54 87 87 87 F F F 20 400-419 54 54 54 87 87 87 F F F 21 420-439 54 54 54 87 87 87 F F F 22 440-459 51 51 51 83 83 83 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 51 51 51 83 83 83 F F F 25 500-519 53 53 53 86 86 86 F F F 26 520-539 54 54 54 87 87 87 F F F 27 540-559 52 52 52 85 85 85 F F F 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 54, Max flush: 87 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.295780117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.341258948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342038414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342085517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342100635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342114633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} {"msg":"Crucible 55b762bd-b3c8-4622-bac0-913ac3f36989 has session id: 9f4a0917-5702-4145-ae32-4a71b7019074","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342707517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Upstairs opts: Upstairs UUID: 55b762bd-b3c8-4622-bac0-913ac3f36989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342749951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Crucible stats registered with UUID: 55b762bd-b3c8-4622-bac0-913ac3f36989","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.342766198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:47.342778347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.3432405Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343266476Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} {"msg":"55b762bd-b3c8-4622-bac0-913ac3f36989 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343335366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343352883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34337545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343389368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343411815Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343436121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343461267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343485614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34351109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343536316Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"ds_connection connected from Ok(127.0.0.1:49506)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343568012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"ds_connection connected from Ok(127.0.0.1:34118)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343740837Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"ds_connection connected from Ok(127.0.0.1:59276)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.343776591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.344222917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.344410849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.344449164Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34481492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.344960659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345224451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345347503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345521548Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[0]R flush_numbers[0..12]: [76, 84, 78, 87, 83, 86, 86, 87, 84, 77, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345768152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[0]R generation[0..12]: [47, 52, 48, 54, 51, 53, 53, 54, 52, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345792398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345807646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[1]R flush_numbers[0..12]: [76, 84, 78, 87, 83, 86, 86, 87, 84, 77, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345824713Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[1]R generation[0..12]: [47, 52, 48, 54, 51, 53, 53, 54, 52, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345838401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345852119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[2]R flush_numbers[0..12]: [76, 84, 78, 87, 83, 86, 86, 87, 84, 77, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345865377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[2]R generation[0..12]: [47, 52, 48, 54, 51, 53, 53, 54, 52, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345878385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345891843Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Max found gen is 55","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345904342Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34591731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345929968Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345942596Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"":"downstairs","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345955334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345973182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"0","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34598625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"1","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.345999028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"client":"2","":"downstairs","session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.346011866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.346025514Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"55b762bd-b3c8-4622-bac0-913ac3f36989 is now active with session: 9f4a0917-5702-4145-ae32-4a71b7019074","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.346038212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.34604986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406,"session_id":"9f4a0917-5702-4145-ae32-4a71b7019074"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:47.346061679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 40960 data: 5 7 7 4 5 4 4 3 4 5 02/30 Read block 100 len 36864 03/30 Flush 04/30 Write block 32 len 32768 data: 8 6 6 5 6 7 7 10 05/30 Read block 472 len 16384 06/30 Read block 407 len 28672 07/30 Read block 238 len 28672 08/30 Read block 347 len 8192 09/30 Write block 428 len 40960 data: 6 6 8 6 4 3 3 6 5 5 10/30 Write block 176 len 36864 data: 4 4 4 4 5 4 5 5 7 11/30 Write block 275 len 4096 data: 7 12/30 Read block 484 len 32768 13/30 Read block 507 len 16384 14/30 Flush 15/30 Write block 361 len 28672 data: 7 8 7 5 4 4 4 16/30 Write block 239 len 8192 data: 6 7 17/30 Read block 217 len 40960 18/30 Read block 139 len 8192 19/30 Read block 152 len 8192 20/30 Flush 21/30 Write block 442 len 32768 data: 4 5 4 4 4 6 4 5 22/30 Read block 289 len 32768 23/30 Write block 389 len 20480 data: 8 8 7 9 9 24/30 Write block 410 len 16384 data: 8 7 9 7 25/30 Write block 40 len 4096 data: 9 26/30 Read block 253 len 28672 27/30 Write block 294 len 40960 data: 10 11 10 7 7 7 5 5 4 3 28/30 Write block 499 len 12288 data: 5 4 5 29/30 Write block 299 len 28672 data: 8 6 6 5 4 5 3 30/30 Read block 568 len 16384 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 10 9 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:47.366697446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 56 56 56 89 89 89 F F F 2 040-059 48 48 48 78 78 78 T T T 3 060-079 54 54 54 87 87 87 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 53 53 53 86 86 86 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 56 56 56 89 89 89 F F F 9 180-199 56 56 56 89 89 89 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 56 56 56 90 90 90 F F F 12 240-259 56 56 56 90 90 90 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 52 52 52 84 84 84 T T T 15 300-319 52 52 52 84 84 84 T T T 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 54 54 54 87 87 87 T T T 20 400-419 54 54 54 87 87 87 T T T 21 420-439 56 56 56 89 89 89 F F F 22 440-459 51 51 51 83 83 83 T T T 23 460-479 53 53 53 86 86 86 F F F 24 480-499 51 51 51 83 83 83 T T T 25 500-519 53 53 53 86 86 86 T T T 26 520-539 54 54 54 87 87 87 F F F 27 540-559 56 56 56 88 88 88 F F F 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 56, Max flush: 90 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 76 76 76 F F F 1 020-039 56 56 56 89 89 89 F F F 2 040-059 57 57 57 91 91 91 F F F 3 060-079 54 54 54 87 87 87 F F F 4 080-099 51 51 51 83 83 83 F F F 5 100-119 53 53 53 86 86 86 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 56 56 56 89 89 89 F F F 9 180-199 56 56 56 89 89 89 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 56 56 56 90 90 90 F F F 12 240-259 56 56 56 90 90 90 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 57 57 57 91 91 91 F F F 20 400-419 57 57 57 91 91 91 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 57 57 57 91 91 91 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 57 57 57 91 91 91 F F F 25 500-519 57 57 57 91 91 91 F F F 26 520-539 54 54 54 87 87 87 F F F 27 540-559 56 56 56 88 88 88 F F F 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 57, Max flush: 91 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.627310196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.672533434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.673345946Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.673397318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.673417735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.673433103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} {"msg":"Crucible edc7cf5a-f551-4212-acc7-d52b303602a7 has session id: c3dd0383-46bc-464c-802e-27050cf71cf4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674028177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Upstairs opts: Upstairs UUID: edc7cf5a-f551-4212-acc7-d52b303602a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674063512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Crucible stats registered with UUID: edc7cf5a-f551-4212-acc7-d52b303602a7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674079229Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:51.674091688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674496209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674520545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} {"msg":"edc7cf5a-f551-4212-acc7-d52b303602a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674585965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674603463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674626859Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674640648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674654506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674679132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674707128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674774898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674807173Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.67483219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56827)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674856726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33546)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.674879973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54700)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.675073594Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.675640362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.675677557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.675707132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.676306815Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.67633875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.676366276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.676888281Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677023271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[0]R flush_numbers[0..12]: [76, 89, 91, 87, 83, 86, 86, 87, 89, 89, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677050037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[0]R generation[0..12]: [47, 56, 57, 54, 51, 53, 53, 54, 56, 56, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677066235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677081042Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[1]R flush_numbers[0..12]: [76, 89, 91, 87, 83, 86, 86, 87, 89, 89, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.6770948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[1]R generation[0..12]: [47, 56, 57, 54, 51, 53, 53, 54, 56, 56, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677108348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677122446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[2]R flush_numbers[0..12]: [76, 89, 91, 87, 83, 86, 86, 87, 89, 89, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677136054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[2]R generation[0..12]: [47, 56, 57, 54, 51, 53, 53, 54, 56, 56, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677149152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677162761Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677175249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677187967Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677205874Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:12:51.677229701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"":"downstairs","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677256767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677275914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"0","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677290242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"1","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.67730327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"client":"2","":"downstairs","session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677316488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677328786Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"edc7cf5a-f551-4212-acc7-d52b303602a7 is now active with session: c3dd0383-46bc-464c-802e-27050cf71cf4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677341364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677352973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414,"session_id":"c3dd0383-46bc-464c-802e-27050cf71cf4"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:51.677364931Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 569 len 40960 data: 6 6 5 6 6 6 6 6 7 6 02/30 Read block 405 len 12288 03/30 Read block 210 len 20480 04/30 Flush 05/30 Flush 06/30 Write block 73 len 4096 data: 5 07/30 Read block 258 len 28672 08/30 Read block 35 len 12288 09/30 Read block 442 len 32768 10/30 Write block 88 len 12288 data: 4 3 3 11/30 Read block 264 len 16384 12/30 Flush 13/30 Write block 496 len 32768 data: 4 5 4 6 5 6 5 8 14/30 Write block 183 len 20480 data: 6 8 7 7 7 15/30 Flush 16/30 Write block 6 len 36864 data: 3 4 5 3 3 3 3 3 4 17/30 Read block 392 len 8192 18/30 Read block 516 len 20480 19/30 Read block 143 len 28672 20/30 Write block 57 len 12288 data: 6 5 4 21/30 Flush 22/30 Write block 103 len 24576 data: 6 6 4 3 5 5 23/30 Read block 69 len 24576 24/30 Write block 32 len 12288 data: 9 7 7 25/30 Write block 422 len 28672 data: 4 3 3 6 5 5 7 26/30 Read block 532 len 28672 27/30 Read block 557 len 36864 28/30 Write block 390 len 12288 data: 9 8 10 29/30 Write block 539 len 8192 data: 5 4 30/30 Read block 111 len 8192 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 7 9 9 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:51.69667768Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 56 56 56 89 89 89 T T T 2 040-059 58 58 58 96 96 96 F F F 3 060-079 58 58 58 94 94 94 F F F 4 080-099 58 58 58 94 94 94 F F F 5 100-119 53 53 53 86 86 86 T T T 6 120-139 53 53 53 86 86 86 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 56 56 56 89 89 89 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 56 56 56 90 90 90 F F F 12 240-259 56 56 56 90 90 90 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 57 57 57 91 91 91 T T T 20 400-419 57 57 57 91 91 91 F F F 21 420-439 56 56 56 89 89 89 T T T 22 440-459 57 57 57 91 91 91 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 54 54 54 87 87 87 T T T 27 540-559 56 56 56 88 88 88 T T T 28 560-579 58 58 58 92 92 92 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 58, Max flush: 96 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 59 59 59 97 97 97 F F F 2 040-059 58 58 58 96 96 96 F F F 3 060-079 58 58 58 94 94 94 F F F 4 080-099 58 58 58 94 94 94 F F F 5 100-119 59 59 59 97 97 97 F F F 6 120-139 53 53 53 86 86 86 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 56 56 56 89 89 89 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 56 56 56 90 90 90 F F F 12 240-259 56 56 56 90 90 90 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 59 59 59 97 97 97 F F F 20 400-419 57 57 57 91 91 91 F F F 21 420-439 59 59 59 97 97 97 F F F 22 440-459 57 57 57 91 91 91 F F F 23 460-479 53 53 53 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 59 59 59 97 97 97 F F F 27 540-559 59 59 59 97 97 97 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 59, Max flush: 97 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.942128418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.986880094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.98752974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.987580602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.98759903Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.987618087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} {"msg":"Crucible 776c995e-a1a7-4db9-ba91-947a60bb9bb7 has session id: 075bbfbb-fd8f-4d62-ba5f-f69edc209c4a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.988223669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Upstairs opts: Upstairs UUID: 776c995e-a1a7-4db9-ba91-947a60bb9bb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.988308546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Crucible stats registered with UUID: 776c995e-a1a7-4db9-ba91-947a60bb9bb7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.988329633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:55.988342952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989009304Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.98904301Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} {"msg":"776c995e-a1a7-4db9-ba91-947a60bb9bb7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989086593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.98910606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989128587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989142565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989157163Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989170671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989184929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989197817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989211195Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989224403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44030)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989368422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53137)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989469337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54364)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.989511771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990076639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990148149Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990284239Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990804843Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990842448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.990875793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991399856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991434631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[0]R flush_numbers[0..12]: [96, 97, 96, 94, 94, 97, 86, 87, 89, 95, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991457208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[0]R generation[0..12]: [58, 59, 58, 58, 58, 59, 53, 54, 56, 58, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991472496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991486664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[1]R flush_numbers[0..12]: [96, 97, 96, 94, 94, 97, 86, 87, 89, 95, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991500192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[1]R generation[0..12]: [58, 59, 58, 58, 58, 59, 53, 54, 56, 58, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.99151366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991527378Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[2]R flush_numbers[0..12]: [96, 97, 96, 94, 94, 97, 86, 87, 89, 95, 87, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991540956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[2]R generation[0..12]: [58, 59, 58, 58, 58, 59, 53, 54, 56, 58, 54, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991554304Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991572242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991585489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991598378Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991610936Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991638492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"":"downstairs","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991663488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991683925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"0","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991707222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"1","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991730628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"client":"2","":"downstairs","session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991757504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991776782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"776c995e-a1a7-4db9-ba91-947a60bb9bb7 is now active with session: 075bbfbb-fd8f-4d62-ba5f-f69edc209c4a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.99179009Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991801898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422,"session_id":"075bbfbb-fd8f-4d62-ba5f-f69edc209c4a"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:12:55.991813906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 465 len 36864 data: 3 4 3 5 4 3 3 4 3 02/30 Read block 513 len 40960 03/30 Write block 522 len 20480 data: 4 3 3 3 3 04/30 Write block 38 len 12288 data: 8 11 10 05/30 Write block 104 len 20480 data: 7 5 4 6 6 06/30 Read block 102 len 4096 07/30 Read block 286 len 28672 08/30 Write block 257 len 40960 data: 6 5 5 4 3 4 5 5 4 6 09/30 Read block 488 len 12288 10/30 Flush 11/30 Flush 12/30 Write block 397 len 20480 data: 9 9 8 8 6 13/30 Read block 90 len 40960 14/30 Flush 15/30 Write block 134 len 4096 data: 5 16/30 Read block 146 len 12288 17/30 Write block 204 len 24576 data: 3 4 4 4 7 7 18/30 Read block 77 len 8192 19/30 Read block 8 len 40960 20/30 Write block 226 len 4096 data: 6 21/30 Read block 219 len 36864 22/30 Read block 416 len 4096 23/30 Flush 24/30 Write block 173 len 4096 data: 5 25/30 Read block 106 len 28672 26/30 Read block 532 len 40960 27/30 Write block 581 len 28672 data: 5 5 3 5 6 6 5 28/30 Write block 156 len 16384 data: 4 5 4 4 29/30 Read block 63 len 32768 30/30 Write block 148 len 40960 data: 7 8 6 6 5 4 3 3 5 6 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:12:56.010363377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 60 60 60 98 98 98 F F F 2 040-059 60 60 60 98 98 98 F F F 3 060-079 58 58 58 94 94 94 F F F 4 080-099 58 58 58 94 94 94 F F F 5 100-119 60 60 60 98 98 98 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 54 54 54 87 87 87 T T T 8 160-179 56 56 56 89 89 89 T T T 9 180-199 58 58 58 95 95 95 F F F 10 200-219 60 60 60 101 101 101 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 60 60 60 98 98 98 F F F 13 260-279 60 60 60 98 98 98 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 60 60 60 100 100 100 F F F 20 400-419 60 60 60 100 100 100 F F F 21 420-439 59 59 59 97 97 97 F F F 22 440-459 57 57 57 91 91 91 F F F 23 460-479 60 60 60 98 98 98 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 59 59 59 97 97 97 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 54 54 54 87 87 87 T T T Max gen: 60, Max flush: 101 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 60 60 60 98 98 98 F F F 2 040-059 60 60 60 98 98 98 F F F 3 060-079 58 58 58 94 94 94 F F F 4 080-099 58 58 58 94 94 94 F F F 5 100-119 60 60 60 98 98 98 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 61 61 61 102 102 102 F F F 8 160-179 61 61 61 102 102 102 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 60 60 60 101 101 101 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 60 60 60 98 98 98 F F F 13 260-279 60 60 60 98 98 98 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 56 56 56 90 90 90 F F F 19 380-399 60 60 60 100 100 100 F F F 20 400-419 60 60 60 100 100 100 F F F 21 420-439 59 59 59 97 97 97 F F F 22 440-459 57 57 57 91 91 91 F F F 23 460-479 60 60 60 98 98 98 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 59 59 59 97 97 97 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 61 61 61 102 102 102 F F F Max gen: 61, Max flush: 102 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.234067769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:00.276757116Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.277116533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.308801062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.309344913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.309388786Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.309406134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.309424191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} {"msg":"Crucible 6acc36a1-badd-4d07-a5b8-d181ca3348cb has session id: 94f6579e-9b65-4733-ad62-90a001a3e92d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310091904Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Upstairs opts: Upstairs UUID: 6acc36a1-badd-4d07-a5b8-d181ca3348cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310132658Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Crucible stats registered with UUID: 6acc36a1-badd-4d07-a5b8-d181ca3348cb","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310149056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:00.310162364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310545498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310569804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} {"msg":"6acc36a1-badd-4d07-a5b8-d181ca3348cb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310633035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310651113Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310673889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310688177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310702925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310716643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310731251Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310744609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310758727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310781914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52697)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.310832096Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41355)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.311093918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62372)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.311132343Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.311486551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.311641868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.311812684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.312145285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.31231765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.312623865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.312787032Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.312908604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[0]R flush_numbers[0..12]: [96, 98, 98, 94, 94, 98, 101, 102, 102, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313224308Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[0]R generation[0..12]: [58, 60, 60, 58, 58, 60, 60, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313244115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313258643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[1]R flush_numbers[0..12]: [96, 98, 98, 94, 94, 98, 101, 102, 102, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.31327677Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[1]R generation[0..12]: [58, 60, 60, 58, 58, 60, 60, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313291019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313304726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[2]R flush_numbers[0..12]: [96, 98, 98, 94, 94, 98, 101, 102, 102, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313318184Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[2]R generation[0..12]: [58, 60, 60, 58, 58, 60, 60, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313331442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.31334526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313358119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313370947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313383545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313406592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"":"downstairs","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313430998Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313467003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"0","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313493609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"1","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313519565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"client":"2","":"downstairs","session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313545791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313574147Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"6acc36a1-badd-4d07-a5b8-d181ca3348cb is now active with session: 94f6579e-9b65-4733-ad62-90a001a3e92d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313600643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.31362398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430,"session_id":"94f6579e-9b65-4733-ad62-90a001a3e92d"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:00.313643257Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 104 len 28672 data: 8 6 5 7 7 5 5 02/30 Write block 442 len 4096 data: 5 03/30 Read block 203 len 12288 04/30 Write block 378 len 8192 data: 5 5 05/30 Read block 198 len 24576 06/30 Read block 348 len 32768 07/30 Write block 183 len 8192 data: 7 9 08/30 Flush 09/30 Write block 179 len 8192 data: 5 6 10/30 Read block 239 len 16384 11/30 Write block 99 len 28672 data: 10 10 10 7 7 9 7 12/30 Write block 418 len 32768 data: 5 5 4 3 5 4 4 7 13/30 Write block 398 len 28672 data: 10 9 9 7 5 7 7 14/30 Read block 407 len 12288 15/30 Read block 574 len 36864 16/30 Write block 539 len 40960 data: 6 5 5 5 4 5 5 6 6 8 17/30 Read block 135 len 8192 18/30 Read block 428 len 4096 19/30 Read block 311 len 24576 20/30 Write block 38 len 24576 data: 9 12 11 7 6 6 21/30 Read block 201 len 28672 22/30 Write block 464 len 12288 data: 4 4 5 23/30 Flush 24/30 Write block 62 len 24576 data: 7 8 8 11 7 5 25/30 Read block 525 len 40960 26/30 Read block 48 len 4096 27/30 Read block 372 len 20480 28/30 Read block 480 len 32768 29/30 Read block 400 len 16384 30/30 Write block 57 len 16384 data: 7 6 5 4 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:00.332470307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 62 62 62 104 104 104 T T T 3 060-079 58 58 58 94 94 94 T T T 4 080-099 62 62 62 104 104 104 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 61 61 61 102 102 102 F F F 8 160-179 62 62 62 104 104 104 F F F 9 180-199 62 62 62 104 104 104 F F F 10 200-219 60 60 60 101 101 101 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 60 60 60 98 98 98 F F F 13 260-279 60 60 60 98 98 98 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 62 62 62 103 103 103 F F F 19 380-399 62 62 62 104 104 104 F F F 20 400-419 62 62 62 104 104 104 F F F 21 420-439 62 62 62 104 104 104 F F F 22 440-459 62 62 62 103 103 103 F F F 23 460-479 62 62 62 104 104 104 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 62 62 62 104 104 104 F F F 27 540-559 62 62 62 104 104 104 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 61 61 61 102 102 102 F F F Max gen: 62, Max flush: 104 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 63 63 63 105 105 105 F F F 3 060-079 63 63 63 105 105 105 F F F 4 080-099 62 62 62 104 104 104 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 61 61 61 102 102 102 F F F 8 160-179 62 62 62 104 104 104 F F F 9 180-199 62 62 62 104 104 104 F F F 10 200-219 60 60 60 101 101 101 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 60 60 60 98 98 98 F F F 13 260-279 60 60 60 98 98 98 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 F F F 18 360-379 62 62 62 103 103 103 F F F 19 380-399 62 62 62 104 104 104 F F F 20 400-419 62 62 62 104 104 104 F F F 21 420-439 62 62 62 104 104 104 F F F 22 440-459 62 62 62 103 103 103 F F F 23 460-479 62 62 62 104 104 104 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 62 62 62 104 104 104 F F F 27 540-559 62 62 62 104 104 104 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 61 61 61 102 102 102 F F F Max gen: 63, Max flush: 105 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.586626477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.631355057Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.63202585Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.632071913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.6320924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.632107978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} {"msg":"Crucible d95cd0d2-aa2c-41da-9935-492817e7d752 has session id: 64f9b26a-a76a-43e9-81d9-fca88fb5a2a1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.632834432Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Upstairs opts: Upstairs UUID: d95cd0d2-aa2c-41da-9935-492817e7d752, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.632875986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Crucible stats registered with UUID: d95cd0d2-aa2c-41da-9935-492817e7d752","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.632899512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:04.632920489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633406699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633431805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} {"msg":"d95cd0d2-aa2c-41da-9935-492817e7d752 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633497636Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633516073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633539019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633553507Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633568135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633585023Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633617638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633644654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.63367063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633695287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:60322)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633718543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62972)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.63374359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61673)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.633767816Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.634484902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.634513198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.634528715Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635164133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635204367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635232313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635781003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635912654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[0]R flush_numbers[0..12]: [96, 104, 105, 105, 104, 104, 101, 102, 104, 104, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.63594105Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[0]R generation[0..12]: [58, 62, 63, 63, 62, 62, 60, 61, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635957567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635972495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[1]R flush_numbers[0..12]: [96, 104, 105, 105, 104, 104, 101, 102, 104, 104, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.635986763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[1]R generation[0..12]: [58, 62, 63, 63, 62, 62, 60, 61, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636000821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636015009Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[2]R flush_numbers[0..12]: [96, 104, 105, 105, 104, 104, 101, 102, 104, 104, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636028607Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[2]R generation[0..12]: [58, 62, 63, 63, 62, 62, 60, 61, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636042275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636056283Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636069231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636082389Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636095497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636108696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B"Wait for a query_work_queue command to finish before sending IO ,"pid":1438,"":"downstairs","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636128233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.63614451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"0","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636158178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"1","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636171246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"client":"2","":"downstairs","session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636184305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636196523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"d95cd0d2-aa2c-41da-9935-492817e7d752 is now active with session: 64f9b26a-a76a-43e9-81d9-fca88fb5a2a1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636209331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636221259Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438,"session_id":"64f9b26a-a76a-43e9-81d9-fca88fb5a2a1"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:04.636233437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 36864 data: 6 7 7 6 7 5 5 4 3 02/30 Write block 419 len 20480 data: 6 5 4 6 5 03/30 Flush 04/30 Write block 104 len 16384 data: 10 8 6 8 05/30 Read block 153 len 16384 06/30 Read block 52 len 24576 07/30 Write block 459 len 20480 data: 6 6 4 4 5 08/30 Write block 512 len 4096 data: 5 09/30 Read block 435 len 20480 10/30 Write block 535 len 20480 data: 5 6 5 6 7 11/30 Write block 407 len 16384 data: 7 7 7 9 12/30 Read block 576 len 8192 13/30 Write block 350 len 28672 data: 6 6 6 7 7 8 9 14/30 Read block 392 len 24576 15/30 Read block 379 len 8192 16/30 Write block 525 len 12288 data: 4 4 4 17/30 Write block 271 len 12288 data: 5 5 5 18/30 Read block 55 len 32768 19/30 Write block 197 len 8192 data: 5 4 20/30 Write block 519 len 24576 data: 9 6 4 5 4 4 21/30 Read block 561 len 4096 22/30 Read block 503 len 24576 23/30 Read block 3 len 40960 24/30 Read block 308 len 4096 25/30 Write block 208 len 24576 data: 8 8 7 7 7 6 26/30 Write block 89 len 12288 data: 4 4 3 27/30 Write block 153 len 40960 data: 5 4 4 6 7 5 5 6 7 7 28/30 Write block 257 len 20480 data: 7 6 6 5 4 29/30 Read block 520 len 8192 30/30 Write block 558 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:04.654138232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 63 63 63 105 105 105 F F F 3 060-079 63 63 63 105 105 105 F F F 4 080-099 62 62 62 104 104 104 T T T 5 100-119 62 62 62 104 104 104 T T T 6 120-139 60 60 60 101 101 101 F F F 7 140-159 61 61 61 102 102 102 T T T 8 160-179 62 62 62 104 104 104 T T T 9 180-199 62 62 62 104 104 104 T T T 10 200-219 60 60 60 101 101 101 T T T 11 220-239 60 60 60 101 101 101 F F F 12 240-259 60 60 60 98 98 98 T T T 13 260-279 60 60 60 98 98 98 T T T 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 54 54 54 87 87 87 T T T 18 360-379 62 62 62 103 103 103 F F F 19 380-399 62 62 62 104 104 104 F F F 20 400-419 64 64 64 106 106 106 T T T 21 420-439 64 64 64 106 106 106 F F F 22 440-459 62 62 62 103 103 103 T T T 23 460-479 62 62 62 104 104 104 T T T 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 T T T 26 520-539 62 62 62 104 104 104 T T T 27 540-559 62 62 62 104 104 104 T T T 28 560-579 58 58 58 92 92 92 F F F 29 580-599 64 64 64 106 106 106 F F F Max gen: 64, Max flush: 106 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 96 96 96 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 63 63 63 105 105 105 F F F 3 060-079 63 63 63 105 105 105 F F F 4 080-099 65 65 65 107 107 107 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 65 65 65 107 107 107 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 65 65 65 107 107 107 F F F 10 200-219 65 65 65 107 107 107 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 65 65 65 107 107 107 F F F 13 260-279 65 65 65 107 107 107 F F F 14 280-299 57 57 57 91 91 91 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 54 54 54 87 87 87 F F F 17 340-359 65 65 65 107 107 107 F F F 18 360-379 62 62 62 103 103 103 F F F 19 380-399 62 62 62 104 104 104 F F F 20 400-419 65 65 65 107 107 107 F F F 21 420-439 64 64 64 106 106 106 F F F 22 440-459 65 65 65 107 107 107 F F F 23 460-479 65 65 65 107 107 107 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 65 65 65 107 107 107 F F F 27 540-559 65 65 65 107 107 107 F F F 28 560-579 58 58 58 92 92 92 F F F 29 580-599 64 64 64 106 106 106 F F F Max gen: 65, Max flush: 107 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.944836923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.98986051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.990585984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.990641796Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.990660234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.990678551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} {"msg":"Crucible 01f6c877-91b6-44c5-8134-732f55f12590 has session id: 603d6ba4-a85c-4c94-a610-e42b03bd13a3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991131085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Upstairs opts: Upstairs UUID: 01f6c877-91b6-44c5-8134-732f55f12590, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991178388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Crucible stats registered with UUID: 01f6c877-91b6-44c5-8134-732f55f12590","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991204264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:08.991228831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991674616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991699382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} {"msg":"01f6c877-91b6-44c5-8134-732f55f12590 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991801927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991821925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991844911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991859249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991873917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991888305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991913871Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991935908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991951036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.991965184Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38858)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.992141148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:36934)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.992178922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35431)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.992213217Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.992925674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.99295342Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.993512208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.993659957Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.993688973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994151985Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994333979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994360595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[0]R flush_numbers[0..12]: [96, 104, 105, 105, 107, 107, 101, 107, 107, 107, 107, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994680149Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[0]R generation[0..12]: [58, 62, 63, 63, 65, 65, 60, 65, 65, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994706115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994722112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[1]R flush_numbers[0..12]: [96, 104, 105, 105, 107, 107, 101, 107, 107, 107, 107, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.99473632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[1]R generation[0..12]: [58, 62, 63, 63, 65, 65, 60, 65, 65, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994750068Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994767476Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[2]R flush_numbers[0..12]: [96, 104, 105, 105, 107, 107, 101, 107, 107, 107, 107, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994781424Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[2]R generation[0..12]: [58, 62, 63, 63, 65, 65, 60, 65, 65, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994794652Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.99480817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994820918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994833916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:13:08.994846754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994868291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"":"downstairs","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994881229Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994893038Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"0","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994905926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"1","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994918594Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"client":"2","":"downstairs","session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994931582Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.99494352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"01f6c877-91b6-44c5-8134-732f55f12590 is now active with session: 603d6ba4-a85c-4c94-a610-e42b03bd13a3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994955978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994967807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446,"session_id":"603d6ba4-a85c-4c94-a610-e42b03bd13a3"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:08.994981795Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 335 len 20480 data: 9 8 10 11 10 02/30 Read block 529 len 32768 03/30 Read block 440 len 32768 04/30 Write block 541 len 12288 data: 6 6 5 05/30 Read block 201 len 24576 06/30 Write block 568 len 40960 data: 6 7 7 6 7 7 7 7 7 8 07/30 Read block 54 len 12288 08/30 Flush 09/30 Write block 470 len 4096 data: 4 10/30 Flush 11/30 Write block 196 len 28672 data: 5 6 5 4 3 3 4 12/30 Write block 265 len 12288 data: 5 7 6 13/30 Write block 356 len 28672 data: 10 9 8 7 7 8 9 14/30 Read block 429 len 28672 15/30 Read block 346 len 24576 16/30 Write block 45 len 40960 data: 4 6 7 6 6 6 7 8 6 6 17/30 Write block 289 len 8192 data: 11 10 18/30 Read block 558 len 20480 19/30 Write block 327 len 24576 data: 6 7 10 8 9 9 20/30 Write block 249 len 8192 data: 5 5 21/30 Read block 308 len 12288 22/30 Read block 124 len 32768 23/30 Write block 12 len 16384 data: 4 4 5 4 24/30 Flush 25/30 Write block 407 len 12288 data: 8 8 8 26/30 Read block 343 len 24576 27/30 Read block 99 len 4096 28/30 Write block 395 len 20480 data: 8 8 10 11 10 29/30 Write block 149 len 32768 data: 9 7 7 6 6 5 5 7 30/30 Write block 553 len 28672 data: 5 4 5 6 4 5 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:09.014505553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 110 110 110 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 66 66 66 110 110 110 F F F 3 060-079 63 63 63 105 105 105 F F F 4 080-099 65 65 65 107 107 107 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 65 65 65 107 107 107 T T T 8 160-179 65 65 65 107 107 107 F F F 9 180-199 66 66 66 110 110 110 F F F 10 200-219 66 66 66 110 110 110 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 66 66 66 110 110 110 F F F 13 260-279 66 66 66 110 110 110 F F F 14 280-299 66 66 66 110 110 110 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 66 66 66 110 110 110 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 66 66 66 110 110 110 F F F 19 380-399 62 62 62 104 104 104 T T T 20 400-419 65 65 65 107 107 107 T T T 21 420-439 64 64 64 106 106 106 F F F 22 440-459 65 65 65 107 107 107 F F F 23 460-479 66 66 66 109 109 109 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 65 65 65 107 107 107 F F F 27 540-559 66 66 66 108 108 108 T T T 28 560-579 66 66 66 108 108 108 F F F 29 580-599 64 64 64 106 106 106 F F F Max gen: 66, Max flush: 110 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 110 110 110 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 66 66 66 110 110 110 F F F 3 060-079 63 63 63 105 105 105 F F F 4 080-099 65 65 65 107 107 107 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 67 67 67 111 111 111 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 66 66 66 110 110 110 F F F 10 200-219 66 66 66 110 110 110 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 66 66 66 110 110 110 F F F 13 260-279 66 66 66 110 110 110 F F F 14 280-299 66 66 66 110 110 110 F F F 15 300-319 57 57 57 91 91 91 F F F 16 320-339 66 66 66 110 110 110 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 66 66 66 110 110 110 F F F 19 380-399 67 67 67 111 111 111 F F F 20 400-419 67 67 67 111 111 111 F F F 21 420-439 64 64 64 106 106 106 F F F 22 440-459 65 65 65 107 107 107 F F F 23 460-479 66 66 66 109 109 109 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 65 65 65 107 107 107 F F F 27 540-559 67 67 67 111 111 111 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 64 64 64 106 106 106 F F F Max gen: 67, Max flush: 111 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.253196625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:13.295744192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.296130426Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.327670595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.328228984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.328271008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.328293315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.328311492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} {"msg":"Crucible 106dc480-0a9c-4ece-a194-542dc7924274 has session id: e0713900-a775-4d2d-a47d-5af35587057f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.328988474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Upstairs opts: Upstairs UUID: 106dc480-0a9c-4ece-a194-542dc7924274, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329028618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Crucible stats registered with UUID: 106dc480-0a9c-4ece-a194-542dc7924274","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329044866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:13.329057494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.32949666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329520586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} {"msg":"106dc480-0a9c-4ece-a194-542dc7924274 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329578518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329596035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329617922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.32963145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329645508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329661786Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329691361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329715928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329731895Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329755452Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33235)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329893532Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50942)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329938065Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64952)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.329963542Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.330769045Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.330795021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.330810249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.331336982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.331565369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.331704668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.331960901Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332076264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[0]R flush_numbers[0..12]: [110, 104, 110, 105, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332224443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[0]R generation[0..12]: [66, 62, 66, 63, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.33224598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332260927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[1]R flush_numbers[0..12]: [110, 104, 110, 105, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332275195Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[1]R generation[0..12]: [66, 62, 66, 63, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332288763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332302541Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[2]R flush_numbers[0..12]: [110, 104, 110, 105, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.33231591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[2]R generation[0..12]: [66, 62, 66, 63, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332329008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332342735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332355434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332368422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.33238107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-01-14T16:13:13.332393738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"":"downstairs","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332418155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332430563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"0","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332443821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"1","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332456559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"client":"2","":"downstairs","session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332469447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332481365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"106dc480-0a9c-4ece-a194-542dc7924274 is now active with session: e0713900-a775-4d2d-a47d-5af35587057f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332493824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.332505282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454,"session_id":"e0713900-a775-4d2d-a47d-5af35587057f"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:13.33251712Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 32768 data: 4 3 5 5 5 5 4 5 02/30 Read block 264 len 32768 03/30 Flush 04/30 Write block 69 len 16384 data: 6 6 5 5 05/30 Write block 264 len 12288 data: 6 6 8 06/30 Flush 07/30 Write block 365 len 32768 data: 5 5 5 4 4 5 5 3 08/30 Write block 310 len 20480 data: 4 4 4 6 5 09/30 Write block 585 len 20480 data: 8 8 7 8 6 10/30 Write block 521 len 40960 data: 5 6 5 5 5 5 5 4 4 5 11/30 Flush 12/30 Write block 9 len 8192 data: 4 4 13/30 Write block 571 len 20480 data: 7 8 8 8 8 14/30 Read block 422 len 40960 15/30 Write block 291 len 8192 data: 9 8 16/30 Read block 208 len 8192 17/30 Read block 38 len 20480 18/30 Flush 19/30 Write block 241 len 36864 data: 4 7 5 5 6 5 5 5 6 20/30 Write block 530 len 24576 data: 6 5 5 4 4 6 21/30 Read block 132 len 28672 22/30 Read block 531 len 20480 23/30 Write block 462 len 36864 data: 5 6 5 5 6 4 6 5 5 24/30 Read block 54 len 32768 25/30 Write block 422 len 4096 data: 7 26/30 Read block 319 len 40960 27/30 Write block 63 len 20480 data: 9 9 12 8 6 28/30 Read block 498 len 4096 29/30 Read block 595 len 4096 30/30 Write block 330 len 8192 data: 9 10 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:13.351418369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 115 115 115 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 66 66 66 110 110 110 F F F 3 060-079 68 68 68 113 113 113 T T T 4 080-099 65 65 65 107 107 107 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 67 67 67 111 111 111 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 66 66 66 110 110 110 F F F 10 200-219 66 66 66 110 110 110 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 66 66 66 110 110 110 T T T 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 66 66 66 110 110 110 T T T 17 340-359 66 66 66 110 110 110 F F F 18 360-379 68 68 68 114 114 114 F F F 19 380-399 67 67 67 111 111 111 F F F 20 400-419 67 67 67 111 111 111 F F F 21 420-439 64 64 64 106 106 106 T T T 22 440-459 65 65 65 107 107 107 F F F 23 460-479 68 68 68 112 112 112 T T T 24 480-499 68 68 68 112 112 112 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 68 68 68 114 114 114 T T T 27 540-559 67 67 67 111 111 111 F F F 28 560-579 68 68 68 115 115 115 F F F 29 580-599 68 68 68 114 114 114 F F F Max gen: 68, Max flush: 115 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 115 115 115 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 66 66 66 110 110 110 F F F 3 060-079 69 69 69 116 116 116 F F F 4 080-099 65 65 65 107 107 107 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 67 67 67 111 111 111 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 66 66 66 110 110 110 F F F 10 200-219 66 66 66 110 110 110 F F F 11 220-239 60 60 60 101 101 101 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 69 69 69 116 116 116 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 68 68 68 114 114 114 F F F 19 380-399 67 67 67 111 111 111 F F F 20 400-419 67 67 67 111 111 111 F F F 21 420-439 69 69 69 116 116 116 F F F 22 440-459 65 65 65 107 107 107 F F F 23 460-479 69 69 69 116 116 116 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 69 69 69 116 116 116 F F F 27 540-559 67 67 67 111 111 111 F F F 28 560-579 68 68 68 115 115 115 F F F 29 580-599 68 68 68 114 114 114 F F F Max gen: 69, Max flush: 116 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.620396112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.664167182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.664842574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.664885217Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.664899965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.664914513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} {"msg":"Crucible 1b89fffd-4e25-4f46-b4e5-3402bd3e2165 has session id: 69ca43c7-cf4c-4ed3-b0bf-171833d07a45","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.665452765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Upstairs opts: Upstairs UUID: 1b89fffd-4e25-4f46-b4e5-3402bd3e2165, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.665544401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Crucible stats registered with UUID: 1b89fffd-4e25-4f46-b4e5-3402bd3e2165","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.665574567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:17.665596614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.66610216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666136155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} {"msg":"1b89fffd-4e25-4f46-b4e5-3402bd3e2165 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666202036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666233141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666272166Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666302711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666333007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666360733Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666387599Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666409176Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666428133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666452729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:37529)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666478565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:65215)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666581181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:59272)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.666620695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.667196661Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.667380934Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.667421808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.667803753Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.667946672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.66802862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668417323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668527447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[0]R flush_numbers[0..12]: [115, 104, 110, 116, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668634402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[0]R generation[0..12]: [68, 62, 66, 69, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668655299Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668670526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[1]R flush_numbers[0..12]: [115, 104, 110, 116, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668684575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[1]R generation[0..12]: [68, 62, 66, 69, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668698402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.6687125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[2]R flush_numbers[0..12]: [115, 104, 110, 116, 107, 107, 101, 111, 107, 110, 110, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668726149Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[2]R generation[0..12]: [68, 62, 66, 69, 65, 65, 60, 67, 65, 66, 66, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668744716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668758884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668771632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.66878453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668797268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668820905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"":"downstairs","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668846321Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668870448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"0","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668896654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"1","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.66892302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"client":"2","":"downstairs","session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668949846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.668974212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"1b89fffd-4e25-4f46-b4e5-3402bd3e2165 is now active with session: 69ca43c7-cf4c-4ed3-b0bf-171833d07a45","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-14T16:13:17.668998799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.669025205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462,"session_id":"69ca43c7-cf4c-4ed3-b0bf-171833d07a45"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:17.669044722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 137 len 40960 data: 2 2 2 2 3 4 4 5 6 5 02/30 Write block 77 len 16384 data: 6 5 6 5 03/30 Read block 530 len 40960 04/30 Read block 491 len 12288 05/30 Write block 202 len 4096 data: 5 06/30 Read block 315 len 8192 07/30 Write block 234 len 16384 data: 7 7 6 5 08/30 Read block 471 len 28672 09/30 Write block 142 len 40960 data: 5 5 6 7 6 7 8 10 8 8 10/30 Read block 558 len 16384 11/30 Flush 12/30 Write block 528 len 28672 data: 5 5 7 6 6 5 5 13/30 Read block 350 len 4096 14/30 Write block 363 len 40960 data: 8 6 6 6 6 5 5 6 6 4 15/30 Write block 584 len 20480 data: 7 9 9 8 9 16/30 Write block 324 len 24576 data: 7 7 6 7 8 11 17/30 Read block 403 len 4096 18/30 Read block 302 len 20480 19/30 Write block 408 len 40960 data: 9 9 10 8 10 8 7 6 6 6 20/30 Read block 433 len 36864 21/30 Read block 484 len 20480 22/30 Read block 12 len 8192 23/30 Write block 194 len 40960 data: 6 5 6 7 6 5 4 4 6 4 24/30 Write block 63 len 28672 data: 10 10 13 9 7 5 7 25/30 Write block 453 len 40960 data: 6 7 7 5 7 6 7 7 5 6 26/30 Read block 517 len 4096 27/30 Write block 49 len 4096 data: 7 28/30 Write block 2 len 16384 data: 5 5 5 4 29/30 Write block 435 len 28672 data: 7 6 6 5 4 4 4 30/30 Write block 454 len 16384 data: 8 8 6 8 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 7 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:17.687741311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 115 115 115 T T T 1 020-039 62 62 62 104 104 104 F F F 2 040-059 66 66 66 110 110 110 T T T 3 060-079 70 70 70 117 117 117 T T T 4 080-099 70 70 70 117 117 117 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 70 70 70 117 117 117 F F F 7 140-159 70 70 70 117 117 117 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 66 66 66 110 110 110 T T T 10 200-219 70 70 70 117 117 117 T T T 11 220-239 70 70 70 117 117 117 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 69 69 69 116 116 116 T T T 17 340-359 66 66 66 110 110 110 F F F 18 360-379 68 68 68 114 114 114 T T T 19 380-399 67 67 67 111 111 111 F F F 20 400-419 67 67 67 111 111 111 T T T 21 420-439 69 69 69 116 116 116 T T T 22 440-459 65 65 65 107 107 107 T T T 23 460-479 69 69 69 116 116 116 T T T 24 480-499 68 68 68 112 112 112 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 69 69 69 116 116 116 T T T 27 540-559 67 67 67 111 111 111 F F F 28 560-579 68 68 68 115 115 115 F F F 29 580-599 68 68 68 114 114 114 T T T Max gen: 70, Max flush: 117 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 62 62 62 104 104 104 F F F 2 040-059 71 71 71 118 118 118 F F F 3 060-079 71 71 71 118 118 118 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 65 65 65 107 107 107 F F F 6 120-139 70 70 70 117 117 117 F F F 7 140-159 70 70 70 117 117 117 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 71 71 71 118 118 118 F F F 10 200-219 71 71 71 118 118 118 F F F 11 220-239 70 70 70 117 117 117 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 71 71 71 118 118 118 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 71 71 71 118 118 118 F F F 19 380-399 67 67 67 111 111 111 F F F 20 400-419 71 71 71 118 118 118 F F F 21 420-439 71 71 71 118 118 118 F F F 22 440-459 71 71 71 118 118 118 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 65 65 65 107 107 107 F F F 26 520-539 71 71 71 118 118 118 F F F 27 540-559 67 67 67 111 111 111 F F F 28 560-579 68 68 68 115 115 115 F F F 29 580-599 71 71 71 118 118 118 F F F Max gen: 71, Max flush: 118 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:21.96615808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.00992602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.010569826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01061119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.010629488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.010644425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} {"msg":"Crucible d7509d48-be71-46db-ab17-5a97fe5732ba has session id: 334f8c82-c1d7-4d66-b2a6-f25579c8be84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011282453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Upstairs opts: Upstairs UUID: d7509d48-be71-46db-ab17-5a97fe5732ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011326776Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Crucible stats registered with UUID: d7509d48-be71-46db-ab17-5a97fe5732ba","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011350253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:22.01137082Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011841951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011868597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} {"msg":"d7509d48-be71-46db-ab17-5a97fe5732ba active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011933018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011951075Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.011980411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012006577Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012033463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01205818Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012086696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012120231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012148946Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012175632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"ds_connection connected from Ok(127.0.0.1:65206)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012200209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"ds_connection connected from Ok(127.0.0.1:63722)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012225395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"ds_connection connected from Ok(127.0.0.1:61631)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012415957Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012835876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.012999163Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.013420721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01356751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.013723737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014034582Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01418534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014317141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[0]R flush_numbers[0..12]: [118, 104, 118, 118, 117, 107, 117, 117, 107, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01480307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[0]R generation[0..12]: [71, 62, 71, 71, 70, 65, 70, 70, 65, 71, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014826847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014842084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[1]R flush_numbers[0..12]: [118, 104, 118, 118, 117, 107, 117, 117, 107, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014856013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[1]R generation[0..12]: [71, 62, 71, 71, 70, 65, 70, 70, 65, 71, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.0148693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014883129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[2]R flush_numbers[0..12]: [118, 104, 118, 118, 117, 107, 117, 117, 107, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014896616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[2]R generation[0..12]: [71, 62, 71, 71, 70, 65, 70, 70, 65, 71, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014909854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014923443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.014935981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.0150087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015037706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015052544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"":"downstairs","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015066212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01508188Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"0","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015095638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"1","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015108726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"client":"2","":"downstairs","session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015121634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015133632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"d7509d48-be71-46db-ab17-5a97fe5732ba is now active with session: 334f8c82-c1d7-4d66-b2a6-f25579c8be84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.01514612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015157849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470,"session_id":"334f8c82-c1d7-4d66-b2a6-f25579c8be84"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:22.015169757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 8192 data: 7 7 02/30 Read block 196 len 28672 03/30 Read block 439 len 4096 04/30 Write block 137 len 36864 data: 3 3 3 3 4 6 6 7 8 05/30 Write block 216 len 4096 data: 8 06/30 Read block 472 len 4096 07/30 Write block 364 len 20480 data: 7 7 7 7 6 08/30 Read block 549 len 32768 09/30 Read block 410 len 20480 10/30 Write block 28 len 16384 data: 5 3 4 5 11/30 Read block 348 len 40960 12/30 Write block 519 len 4096 data: 10 13/30 Read block 6 len 16384 14/30 Write block 198 len 40960 data: 7 6 5 5 7 5 4 5 5 5 15/30 Read block 289 len 28672 16/30 Write block 418 len 36864 data: 6 7 6 5 8 6 5 8 6 17/30 Write block 540 len 24576 data: 6 7 7 6 6 6 18/30 Read block 215 len 40960 19/30 Read block 16 len 4096 20/30 Read block 59 len 40960 21/30 Write block 107 len 28672 data: 9 8 6 6 5 5 4 22/30 Write block 141 len 36864 data: 5 7 7 8 9 7 8 9 11 23/30 Write block 118 len 12288 data: 3 3 5 24/30 Read block 351 len 36864 25/30 Read block 204 len 40960 26/30 Write block 555 len 36864 data: 6 7 5 6 6 4 5 5 5 27/30 Write block 149 len 40960 data: 12 9 9 7 7 6 6 8 8 6 28/30 Read block 254 len 8192 29/30 Read block 318 len 20480 30/30 Write block 113 len 16384 data: 5 4 3 3 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:22.034672518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 62 62 62 104 104 104 T T T 2 040-059 71 71 71 118 118 118 F F F 3 060-079 71 71 71 118 118 118 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 65 65 65 107 107 107 T T T 6 120-139 70 70 70 117 117 117 T T T 7 140-159 70 70 70 117 117 117 T T T 8 160-179 65 65 65 107 107 107 F F F 9 180-199 71 71 71 118 118 118 T T T 10 200-219 71 71 71 118 118 118 T T T 11 220-239 70 70 70 117 117 117 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 71 71 71 118 118 118 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 71 71 71 118 118 118 T T T 19 380-399 67 67 67 111 111 111 F F F 20 400-419 71 71 71 118 118 118 T T T 21 420-439 71 71 71 118 118 118 T T T 22 440-459 71 71 71 118 118 118 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 65 65 65 107 107 107 T T T 26 520-539 71 71 71 118 118 118 F F F 27 540-559 67 67 67 111 111 111 T T T 28 560-579 68 68 68 115 115 115 T T T 29 580-599 71 71 71 118 118 118 F F F Max gen: 71, Max flush: 118 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 72 72 72 119 119 119 F F F 2 040-059 71 71 71 118 118 118 F F F 3 060-079 71 71 71 118 118 118 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 119 119 119 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 72 72 72 119 119 119 F F F 10 200-219 72 72 72 119 119 119 F F F 11 220-239 70 70 70 117 117 117 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 114 114 114 F F F 16 320-339 71 71 71 118 118 118 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 67 67 67 111 111 111 F F F 20 400-419 72 72 72 119 119 119 F F F 21 420-439 72 72 72 119 119 119 F F F 22 440-459 71 71 71 118 118 118 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 72 72 72 119 119 119 F F F 26 520-539 71 71 71 118 118 118 F F F 27 540-559 72 72 72 119 119 119 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 71 71 71 118 118 118 F F F Max gen: 72, Max flush: 119 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.315667272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.360664693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.36144397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.361489233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.361504341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.361519889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} {"msg":"Crucible 04e82e88-ce92-4686-ac7e-f35e489da626 has session id: 04e6e60d-5861-4f86-bcd6-21dfd94b4c24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.36212995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Upstairs opts: Upstairs UUID: 04e82e88-ce92-4686-ac7e-f35e489da626, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362164285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Crucible stats registered with UUID: 04e82e88-ce92-4686-ac7e-f35e489da626","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362183952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:26.36219747Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362562387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362586544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} {"msg":"04e82e88-ce92-4686-ac7e-f35e489da626 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362648615Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362666022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362687999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362701437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362715365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362728373Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362742301Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362758979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362783585Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362809421Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"ds_connection connected from Ok(127.0.0.1:62898)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362837977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"ds_connection connected from Ok(127.0.0.1:43848)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362882041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"ds_connection connected from Ok(127.0.0.1:56689)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.362938742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.363698462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.363735107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.363811646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.364292556Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.36439758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.364554647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.364891348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.364963588Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[0]R flush_numbers[0..12]: [118, 119, 118, 118, 117, 119, 119, 119, 107, 119, 119, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365074902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[0]R generation[0..12]: [71, 72, 71, 71, 70, 72, 72, 72, 65, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365094679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365110097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[1]R flush_numbers[0..12]: [118, 119, 118, 118, 117, 119, 119, 119, 107, 119, 119, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365123855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[1]R generation[0..12]: [71, 72, 71, 71, 70, 72, 72, 72, 65, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365137242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.36515439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[2]R flush_numbers[0..12]: [118, 119, 118, 118, 117, 119, 119, 119, 107, 119, 119, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365168198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[2]R generation[0..12]: [71, 72, 71, 71, 70, 72, 72, 72, 65, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365181366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365194954Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365207402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365220181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365232839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365245197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"":"downstairs","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365258235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365270033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid"Wait for a query_work_queue command to finish before sending IO :1478,"client":"0","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365298119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"1","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365312387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"client":"2","":"downstairs","session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365325395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365337683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"04e82e88-ce92-4686-ac7e-f35e489da626 is now active with session: 04e6e60d-5861-4f86-bcd6-21dfd94b4c24","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365350462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365364289Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478,"session_id":"04e6e60d-5861-4f86-bcd6-21dfd94b4c24"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:26.365376338Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 505 len 40960 data: 9 10 11 11 9 7 6 6 5 6 02/30 Write block 294 len 16384 data: 11 12 11 8 03/30 Flush 04/30 Write block 304 len 24576 data: 6 4 4 4 4 4 05/30 Read block 509 len 32768 06/30 Read block 57 len 4096 07/30 Read block 479 len 24576 08/30 Write block 35 len 28672 data: 6 7 8 10 13 12 8 09/30 Read block 173 len 40960 10/30 Read block 201 len 8192 11/30 Read block 33 len 12288 12/30 Write block 526 len 20480 data: 6 6 6 6 8 13/30 Read block 523 len 4096 14/30 Flush 15/30 Write block 284 len 24576 data: 9 11 10 11 11 12 16/30 Write block 581 len 16384 data: 6 6 4 8 17/30 Read block 440 len 12288 18/30 Read block 390 len 40960 19/30 Read block 476 len 24576 20/30 Read block 116 len 32768 21/30 Read block 158 len 24576 22/30 Read block 379 len 28672 23/30 Read block 495 len 12288 24/30 Read block 580 len 12288 25/30 Write block 528 len 40960 data: 7 7 9 7 7 6 6 7 7 6 26/30 Read block 175 len 28672 27/30 Write block 389 len 40960 data: 9 10 9 11 10 7 9 9 11 12 28/30 Write block 535 len 36864 data: 8 8 7 7 8 7 8 8 7 29/30 Read block 514 len 8192 30/30 Write block 232 len 20480 data: 8 6 8 8 7 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 6 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:26.385909709Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 74 74 74 121 121 121 F F F 3 060-079 71 71 71 118 118 118 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 119 119 119 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 72 72 72 119 119 119 F F F 10 200-219 72 72 72 119 119 119 F F F 11 220-239 70 70 70 117 117 117 T T T 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 74 74 74 120 120 120 T T T 15 300-319 74 74 74 121 121 121 F F F 16 320-339 71 71 71 118 118 118 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 67 67 67 111 111 111 T T T 20 400-419 72 72 72 119 119 119 F F F 21 420-439 72 72 72 119 119 119 F F F 22 440-459 71 71 71 118 118 118 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 74 74 74 121 121 121 T T T 27 540-559 72 72 72 119 119 119 T T T 28 560-579 72 72 72 119 119 119 F F F 29 580-599 71 71 71 118 118 118 T T T Max gen: 74, Max flush: 121 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 74 74 74 121 121 121 F F F 3 060-079 71 71 71 118 118 118 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 119 119 119 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 72 72 72 119 119 119 F F F 10 200-219 72 72 72 119 119 119 F F F 11 220-239 75 75 75 122 122 122 F F F 12 240-259 69 69 69 116 116 116 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 75 75 75 122 122 122 F F F 15 300-319 74 74 74 121 121 121 F F F 16 320-339 71 71 71 118 118 118 F F F 17 340-359 66 66 66 110 110 110 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 75 75 75 122 122 122 F F F 20 400-419 72 72 72 119 119 119 F F F 21 420-439 72 72 72 119 119 119 F F F 22 440-459 71 71 71 118 118 118 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 75 75 75 122 122 122 F F F 27 540-559 75 75 75 122 122 122 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 75, Max flush: 122 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.627779648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.672467494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673097772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673169412Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673202847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673235122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} {"msg":"Crucible d4e7c026-c315-48a1-ac2c-9a2c8aacefc1 has session id: d5bb3d24-fa57-41d1-8a12-ac2073f23494","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673734729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Upstairs opts: Upstairs UUID: d4e7c026-c315-48a1-ac2c-9a2c8aacefc1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673780153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Crucible stats registered with UUID: d4e7c026-c315-48a1-ac2c-9a2c8aacefc1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.673805459Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:30.673828036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674193642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674219959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} {"msg":"d4e7c026-c315-48a1-ac2c-9a2c8aacefc1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674276641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674295248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674318084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674332182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.67434853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674412941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674445506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674470122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674495539Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674520655Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"ds_connection connected from Ok(127.0.0.1:62181)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674546451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"ds_connection connected from Ok(127.0.0.1:53840)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674574757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"ds_connection connected from Ok(127.0.0.1:56066)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.674604073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.675191557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.675328667Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.675414665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.675818006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.675893805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676118593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676440395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676462572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[0]R flush_numbers[0..12]: [118, 121, 121, 118, 117, 119, 119, 119, 107, 119, 119, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676646965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[0]R generation[0..12]: [71, 74, 74, 71, 70, 72, 72, 72, 65, 72, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676668002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.6766827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[1]R flush_numbers[0..12]: [118, 121, 121, 118, 117, 119, 119, 119, 107, 119, 119, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676697128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[1]R generation[0..12]: [71, 74, 74, 71, 70, 72, 72, 72, 65, 72, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676710716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676724334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[2]R flush_numbers[0..12]: [118, 121, 121, 118, 117, 119, 119, 119, 107, 119, 119, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676739602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[2]R generation[0..12]: [71, 74, 74, 71, 70, 72, 72, 72, 65, 72, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676763219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676789115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676808202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676830369Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676860134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.67688958Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"":"downstairs","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676906788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676918896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"0","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676939033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"1","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.67696238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"client":"2","":"downstairs","session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.676987196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.677003114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"d4e7c026-c315-48a1-ac2c-9a2c8aacefc1 is now active with session: d5bb3d24-fa57-41d1-8a12-ac2073f23494","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.677015882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.67702756Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486,"session_id":"d5bb3d24-fa57-41d1-8a12-ac2073f23494"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:30.677046737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 48 len 16384 data: 7 8 7 8 02/30 Write block 134 len 24576 data: 6 4 4 4 4 4 03/30 Read block 474 len 28672 04/30 Write block 537 len 16384 data: 8 8 9 8 05/30 Write block 421 len 20480 data: 6 9 7 6 9 06/30 Read block 461 len 36864 07/30 Write block 443 len 8192 data: 6 5 08/30 Read block 270 len 20480 09/30 Read block 124 len 8192 10/30 Read block 439 len 36864 11/30 Read block 308 len 16384 12/30 Write block 236 len 28672 data: 8 6 6 7 8 5 8 13/30 Write block 66 len 36864 data: 10 8 6 8 7 6 6 6 5 14/30 Flush 15/30 Write block 555 len 16384 data: 7 8 6 7 16/30 Write block 193 len 40960 data: 7 7 6 7 8 8 7 6 6 8 17/30 Read block 276 len 36864 18/30 Write block 355 len 4096 data: 9 19/30 Read block 117 len 24576 20/30 Read block 61 len 40960 21/30 Read block 533 len 24576 22/30 Write block 349 len 24576 data: 8 7 7 7 8 8 23/30 Write block 69 len 28672 data: 9 8 7 7 7 6 6 24/30 Write block 520 len 8192 data: 7 6 25/30 Write block 326 len 16384 data: 7 8 9 12 26/30 Read block 61 len 12288 27/30 Read block 328 len 32768 28/30 Read block 303 len 24576 29/30 Write block 51 len 20480 data: 9 9 7 7 6 30/30 Read block 528 len 20480 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 4 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 14 16 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:30.696621509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 76 76 76 123 123 123 T T T 3 060-079 76 76 76 123 123 123 T T T 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 72 72 72 119 119 119 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 72 72 72 119 119 119 T T T 10 200-219 72 72 72 119 119 119 T T T 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 75 75 75 122 122 122 F F F 15 300-319 74 74 74 121 121 121 F F F 16 320-339 71 71 71 118 118 118 T T T 17 340-359 66 66 66 110 110 110 T T T 18 360-379 72 72 72 119 119 119 F F F 19 380-399 75 75 75 122 122 122 F F F 20 400-419 72 72 72 119 119 119 F F F 21 420-439 76 76 76 123 123 123 F F F 22 440-459 76 76 76 123 123 123 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 76 76 76 123 123 123 T T T 27 540-559 76 76 76 123 123 123 T T T 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 76, Max flush: 123 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 72 72 72 119 119 119 F F F 8 160-179 65 65 65 107 107 107 F F F 9 180-199 77 77 77 124 124 124 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 68 68 68 113 113 113 F F F 14 280-299 75 75 75 122 122 122 F F F 15 300-319 74 74 74 121 121 121 F F F 16 320-339 77 77 77 124 124 124 F F F 17 340-359 77 77 77 124 124 124 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 75 75 75 122 122 122 F F F 20 400-419 72 72 72 119 119 119 F F F 21 420-439 76 76 76 123 123 123 F F F 22 440-459 76 76 76 123 123 123 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 77 77 77 124 124 124 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 77, Max flush: 124 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.948474134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.992975477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.993591657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.993633821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.993648439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.993662677Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} {"msg":"Crucible 4bb4eb9a-81a1-40d3-a4a5-6b13741ca68b has session id: e19bdc97-5448-4799-a73e-e13fd7763a35","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994253141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Upstairs opts: Upstairs UUID: 4bb4eb9a-81a1-40d3-a4a5-6b13741ca68b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994291605Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Crucible stats registered with UUID: 4bb4eb9a-81a1-40d3-a4a5-6b13741ca68b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994307323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:34.994322991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994755528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994779154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} {"msg":"4bb4eb9a-81a1-40d3-a4a5-6b13741ca68b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994828967Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994855053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994885349Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994912375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994936072Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994962448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.994989554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.99501547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995041826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995066533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:41663)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995091009Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:59644)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995117145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:52149)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995246986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995705899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.995745304Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.996378671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.996418846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.996450681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.996981704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997019128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997062242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[0]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 119, 107, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997496439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[0]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 72, 65, 77, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997517096Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997531914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[1]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 119, 107, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997545992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[1]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 72, 65, 77, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.99756055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997574457Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[2]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 119, 107, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997591445Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[2]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 72, 65, 77, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997605003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997618581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997631279Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:13:34.997644327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997661165Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997674003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"":"downstairs","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997686711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997698279Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"0","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997711008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"1","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997723626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"client":"2","":"downstairs","session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997736454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997748322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"4bb4eb9a-81a1-40d3-a4a5-6b13741ca68b is now active with session: e19bdc97-5448-4799-a73e-e13fd7763a35","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.99776102Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997776218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494,"session_id":"e19bdc97-5448-4799-a73e-e13fd7763a35"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:34.997790696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 24576 data: 7 7 9 9 9 9 02/30 Read block 284 len 32768 03/30 Read block 21 len 36864 04/30 Read block 424 len 12288 05/30 Read block 255 len 32768 06/30 Read block 321 len 32768 07/30 Write block 325 len 24576 data: 8 8 9 10 13 10 08/30 Read block 393 len 24576 09/30 Write block 173 len 36864 data: 6 5 4 5 5 5 6 7 5 10/30 Read block 201 len 32768 11/30 Write block 345 len 16384 data: 8 8 7 7 12/30 Write block 145 len 28672 data: 10 8 9 10 13 10 10 13/30 Write block 143 len 12288 data: 8 9 11 14/30 Write block 441 len 32768 data: 5 6 7 6 5 5 7 5 15/30 Write block 279 len 16384 data: 6 6 7 7 16/30 Read block 84 len 32768 17/30 Write block 354 len 20480 data: 9 10 11 10 9 18/30 Write block 523 len 24576 data: 6 6 6 7 7 8 19/30 Write block 183 len 28672 data: 8 10 8 8 8 7 7 20/30 Read block 396 len 28672 21/30 Read block 407 len 36864 22/30 Read block 75 len 4096 23/30 Read block 523 len 32768 24/30 Read block 256 len 8192 25/30 Write block 399 len 32768 data: 11 10 8 6 8 8 9 9 26/30 Read block 509 len 16384 27/30 Read block 400 len 32768 28/30 Read block 111 len 28672 29/30 Read block 364 len 40960 30/30 Read block 163 len 16384 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:35.018186997Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 72 72 72 119 119 119 T T T 8 160-179 65 65 65 107 107 107 T T T 9 180-199 77 77 77 124 124 124 T T T 10 200-219 77 77 77 124 124 124 F F F 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 68 68 68 113 113 113 T T T 14 280-299 75 75 75 122 122 122 T T T 15 300-319 74 74 74 121 121 121 F F F 16 320-339 77 77 77 124 124 124 T T T 17 340-359 77 77 77 124 124 124 T T T 18 360-379 72 72 72 119 119 119 F F F 19 380-399 75 75 75 122 122 122 T T T 20 400-419 72 72 72 119 119 119 T T T 21 420-439 76 76 76 123 123 123 F F F 22 440-459 76 76 76 123 123 123 T T T 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 77 77 77 124 124 124 T T T 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 77, Max flush: 124 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 118 118 118 F F F 1 020-039 74 74 74 121 121 121 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 78 78 78 125 125 125 F F F 9 180-199 78 78 78 125 125 125 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 78 78 78 125 125 125 F F F 14 280-299 78 78 78 125 125 125 F F F 15 300-319 74 74 74 121 121 121 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 78 78 78 125 125 125 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 78 78 78 125 125 125 F F F 20 400-419 78 78 78 125 125 125 F F F 21 420-439 76 76 76 123 123 123 F F F 22 440-459 78 78 78 125 125 125 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 68 68 68 112 112 112 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 78 78 78 125 125 125 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 78, Max flush: 125 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.292576093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.337379073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338086039Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338138002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338156699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338174237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} {"msg":"Crucible 0854e7f2-f871-4310-a699-907a2fec55fc has session id: 7f1ebead-a6b5-4893-b8e7-05f933ced07b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338718167Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Upstairs opts: Upstairs UUID: 0854e7f2-f871-4310-a699-907a2fec55fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338765871Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Crucible stats registered with UUID: 0854e7f2-f871-4310-a699-907a2fec55fc","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.338789007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:39.338808994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339307972Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339334898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} {"msg":"0854e7f2-f871-4310-a699-907a2fec55fc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339398528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339417046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339439593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.33945818Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339472868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339485606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339499204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339587931Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339614227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339638564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41526)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.33966589Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50878)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339793901Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52505)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.339830696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.340312975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.340341191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.34048802Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.34097122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341016813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341048279Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341557574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341581201Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[0]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341636443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[0]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.34165487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341675797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[1]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341689695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[1]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341702923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341716631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[2]R flush_numbers[0..12]: [118, 121, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341730429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[2]R generation[0..12]: [71, 74, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341743727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341757165Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341769823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341782581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341798739Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341811687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"":"downstairs","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341824535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"Wait for a query_work_queue command to finish before sending IO 7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341844203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"0","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341857191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"1","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341869719Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"client":"2","":"downstairs","session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341882647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341894635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"0854e7f2-f871-4310-a699-907a2fec55fc is now active with session: 7f1ebead-a6b5-4893-b8e7-05f933ced07b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341907063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.341918682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502,"session_id":"7f1ebead-a6b5-4893-b8e7-05f933ced07b"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:39.3419306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 12288 data: 7 5 5 02/30 Flush 03/30 Write block 532 len 32768 data: 8 8 8 10 10 10 10 10 04/30 Read block 470 len 36864 05/30 Flush 06/30 Write block 315 len 4096 data: 4 07/30 Write block 520 len 24576 data: 8 7 7 7 7 7 08/30 Read block 376 len 36864 09/30 Write block 270 len 32768 data: 6 6 6 6 7 8 7 8 10/30 Read block 356 len 32768 11/30 Write block 317 len 8192 data: 4 3 12/30 Read block 579 len 4096 13/30 Read block 568 len 20480 14/30 Read block 169 len 4096 15/30 Read block 334 len 36864 16/30 Read block 573 len 28672 17/30 Read block 164 len 4096 18/30 Read block 25 len 20480 19/30 Read block 291 len 32768 20/30 Read block 536 len 4096 21/30 Read block 401 len 12288 22/30 Write block 15 len 8192 data: 5 2 23/30 Flush 24/30 Write block 446 len 28672 data: 6 8 6 6 5 5 5 25/30 Write block 28 len 32768 data: 6 4 5 6 10 8 8 7 26/30 Write block 535 len 16384 data: 11 11 11 11 27/30 Read block 323 len 8192 28/30 Write block 510 len 36864 data: 8 7 7 6 7 6 6 7 8 29/30 Read block 310 len 40960 30/30 Read block 236 len 8192 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:39.361883636Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 74 74 74 121 121 121 T T T 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 78 78 78 125 125 125 F F F 9 180-199 78 78 78 125 125 125 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 80 80 80 128 128 128 F F F 14 280-299 78 78 78 125 125 125 F F F 15 300-319 80 80 80 128 128 128 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 78 78 78 125 125 125 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 78 78 78 125 125 125 F F F 20 400-419 78 78 78 125 125 125 F F F 21 420-439 76 76 76 123 123 123 F F F 22 440-459 78 78 78 125 125 125 T T T 23 460-479 71 71 71 118 118 118 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 74 74 74 120 120 120 T T T 26 520-539 80 80 80 128 128 128 T T T 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 80, Max flush: 128 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 78 78 78 125 125 125 F F F 9 180-199 78 78 78 125 125 125 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 76 76 76 123 123 123 F F F 12 240-259 76 76 76 123 123 123 F F F 13 260-279 80 80 80 128 128 128 F F F 14 280-299 78 78 78 125 125 125 F F F 15 300-319 80 80 80 128 128 128 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 78 78 78 125 125 125 F F F 18 360-379 72 72 72 119 119 119 F F F 19 380-399 78 78 78 125 125 125 F F F 20 400-419 78 78 78 125 125 125 F F F 21 420-439 76 76 76 123 123 123 F F F 22 440-459 81 81 81 129 129 129 F F F 23 460-479 71 71 71 118 118 118 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 81 81 81 129 129 129 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 81, Max flush: 129 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.596695762Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.641421193Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642080367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.64212425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642140458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642154896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} {"msg":"Crucible f74e3664-d6f4-4e29-963e-146d64277c15 has session id: 3b13a2c6-fdb3-423e-8cd6-a1902df048df","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642760158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Upstairs opts: Upstairs UUID: f74e3664-d6f4-4e29-963e-146d64277c15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642801752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Crucible stats registered with UUID: f74e3664-d6f4-4e29-963e-146d64277c15","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.642824108Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:43.642845785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643386637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643410883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} {"msg":"f74e3664-d6f4-4e29-963e-146d64277c15 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643470794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643489162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643511758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643525886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643540044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643552822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643566611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643631751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643664996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643692962Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"ds_connection connected from Ok(127.0.0.1:46375)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643720238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"ds_connection connected from Ok(127.0.0.1:38652)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643747134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"ds_connection connected from Ok(127.0.0.1:59176)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.643853469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.644373363Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.644400069Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.644566275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645033287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645063553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.64507973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.64556423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645683842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[0]R flush_numbers[0..12]: [128, 129, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645722057Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[0]R generation[0..12]: [80, 81, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645738004Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645752722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[1]R flush_numbers[0..12]: [128, 129, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.64576674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[1]R generation[0..12]: [80, 81, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645780178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645794126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[2]R flush_numbers[0..12]: [128, 129, 124, 124, 117, 119, 123, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645811184Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[2]R generation[0..12]: [80, 81, 77, 77, 70, 72, 76, 78, 78, 78, 77, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645827941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645852818Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645877864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645896311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645914869Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645943354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"":"downstairs","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645968911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.645989498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"0","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646018754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"1","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646032332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"client":"2","":"downstairs","session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.64604538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646057508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"f74e3664-d6f4-4e29-963e-146d64277c15 is now active with session: 3b13a2c6-fdb3-423e-8cd6-a1902df048df","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646070136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646081794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510,"session_id":"3b13a2c6-fdb3-423e-8cd6-a1902df048df"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:43.646093683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 40960 data: 6 7 6 9 9 10 11 12 12 10 02/30 Write block 372 len 36864 data: 5 4 5 5 4 6 6 6 4 03/30 Flush 04/30 Write block 420 len 20480 data: 7 7 10 8 7 05/30 Write block 166 len 8192 data: 7 6 06/30 Read block 352 len 20480 07/30 Write block 236 len 28672 data: 9 7 7 8 9 6 9 08/30 Flush 09/30 Write block 169 len 40960 data: 7 5 5 5 7 6 5 6 6 6 10/30 Read block 567 len 8192 11/30 Write block 345 len 40960 data: 9 9 8 8 9 8 8 8 9 10 12/30 Read block 346 len 40960 13/30 Read block 435 len 12288 14/30 Read block 328 len 16384 15/30 Read block 119 len 40960 16/30 Read block 188 len 20480 17/30 Read block 291 len 24576 18/30 Read block 346 len 16384 19/30 Flush 20/30 Write block 89 len 4096 data: 5 21/30 Write block 507 len 4096 data: 13 22/30 Read block 390 len 20480 23/30 Write block 134 len 4096 data: 7 24/30 Write block 418 len 12288 data: 7 8 8 25/30 Write block 303 len 40960 data: 5 7 5 5 5 5 5 5 5 5 26/30 Write block 236 len 4096 data: 10 27/30 Write block 272 len 16384 data: 7 7 8 9 28/30 Write block 279 len 12288 data: 7 7 8 29/30 Write block 273 len 24576 data: 8 9 10 8 9 8 30/30 Write block 470 len 40960 data: 6 4 5 5 4 6 6 6 6 5 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:43.66452339Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 70 70 70 117 117 117 T T T 5 100-119 72 72 72 119 119 119 F F F 6 120-139 76 76 76 123 123 123 T T T 7 140-159 78 78 78 125 125 125 F F F 8 160-179 82 82 82 132 132 132 F F F 9 180-199 78 78 78 125 125 125 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 82 82 82 131 131 131 T T T 12 240-259 82 82 82 131 131 131 F F F 13 260-279 80 80 80 128 128 128 T T T 14 280-299 78 78 78 125 125 125 T T T 15 300-319 80 80 80 128 128 128 T T T 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 82 82 82 130 130 130 F F F 19 380-399 82 82 82 130 130 130 F F F 20 400-419 78 78 78 125 125 125 T T T 21 420-439 82 82 82 131 131 131 T T T 22 440-459 81 81 81 129 129 129 F F F 23 460-479 71 71 71 118 118 118 T T T 24 480-499 80 80 80 126 126 126 F F F 25 500-519 82 82 82 130 130 130 T T T 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 82, Max flush: 132 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 83 83 83 133 133 133 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 82 82 82 132 132 132 F F F 9 180-199 78 78 78 125 125 125 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 F F F 12 240-259 82 82 82 131 131 131 F F F 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 83 83 83 133 133 133 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 82 82 82 130 130 130 F F F 19 380-399 82 82 82 130 130 130 F F F 20 400-419 83 83 83 133 133 133 F F F 21 420-439 83 83 83 133 133 133 F F F 22 440-459 81 81 81 129 129 129 F F F 23 460-479 83 83 83 133 133 133 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 83 83 83 133 133 133 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 83, Max flush: 133 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.929370585Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.974248883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.974858315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.974905988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.974923555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.974946552Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} {"msg":"Crucible bb620baa-26fb-4a49-ae79-c4122cfa1701 has session id: 66a54098-cbcb-4d1d-8eb4-df52129615f5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.975547044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Upstairs opts: Upstairs UUID: bb620baa-26fb-4a49-ae79-c4122cfa1701, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.975592198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Crucible stats registered with UUID: bb620baa-26fb-4a49-ae79-c4122cfa1701","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.975615125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:47.975635531Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976144448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.9761982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} {"msg":"bb620baa-26fb-4a49-ae79-c4122cfa1701 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976274089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976294646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.97633102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976358966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976387072Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976416078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976444284Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976475029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976502435Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976529441Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35192)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976644145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61195)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976682459Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58059)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.976710455Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977164709Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977191655Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977220621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977721748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977753463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.977978421Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978314932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978352526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[0]R flush_numbers[0..12]: [128, 129, 124, 124, 133, 119, 133, 125, 132, 125, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978485337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[0]R generation[0..12]: [80, 81, 77, 77, 83, 72, 83, 78, 82, 78, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978510203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978525631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[1]R flush_numbers[0..12]: [128, 129, 124, 124, 133, 119, 133, 125, 132, 125, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978539529Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[1]R generation[0..12]: [80, 81, 77, 77, 83, 72, 83, 78, 82, 78, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978553217Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978566865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[2]R flush_numbers[0..12]: [128, 129, 124, 124, 133, 119, 133, 125, 132, 125, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978580213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[2]R generation[0..12]: [80, 81, 77, 77, 83, 72, 83, 78, 82, 78, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978593311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978606899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978619437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978632275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:13:47.978644773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.97866978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"":"downstairs","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978696356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978720013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"0","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978749668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"1","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978776984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"client":"2","":"downstairs","session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.97880361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978828347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"bb620baa-26fb-4a49-ae79-c4122cfa1701 is now active with session: 66a54098-cbcb-4d1d-8eb4-df52129615f5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978857023Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978881179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518,"session_id":"66a54098-cbcb-4d1d-8eb4-df52129615f5"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:47.978894707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 173 len 4096 data: 8 02/30 Read block 134 len 36864 03/30 Read block 408 len 12288 04/30 Write block 433 len 16384 data: 4 4 8 7 05/30 Write block 312 len 28672 data: 6 7 6 5 4 5 4 06/30 Read block 112 len 8192 07/30 Read block 196 len 28672 08/30 Read block 142 len 24576 09/30 Read block 529 len 12288 10/30 Read block 334 len 36864 11/30 Write block 451 len 24576 data: 6 6 7 9 9 7 12/30 Read block 38 len 8192 13/30 Read block 266 len 16384 14/30 Read block 514 len 36864 15/30 Read block 385 len 4096 16/30 Read block 272 len 32768 17/30 Flush 18/30 Write block 0 len 16384 data: 4 5 6 6 19/30 Write block 254 len 4096 data: 5 20/30 Read block 537 len 12288 21/30 Read block 216 len 28672 22/30 Flush 23/30 Write block 198 len 4096 data: 9 24/30 Read block 469 len 12288 25/30 Read block 373 len 20480 26/30 Read block 477 len 12288 27/30 Read block 262 len 8192 28/30 Read block 166 len 8192 29/30 Read block 205 len 4096 30/30 Read block 498 len 4096 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:47.99688035Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 83 83 83 133 133 133 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 78 78 78 125 125 125 T T T 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 F F F 12 240-259 84 84 84 135 135 135 F F F 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 84 84 84 134 134 134 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 82 82 82 130 130 130 F F F 19 380-399 82 82 82 130 130 130 F F F 20 400-419 83 83 83 133 133 133 F F F 21 420-439 84 84 84 134 134 134 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 83 83 83 133 133 133 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 83 83 83 133 133 133 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 84, Max flush: 135 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 77 77 77 124 124 124 F F F 3 060-079 77 77 77 124 124 124 F F F 4 080-099 83 83 83 133 133 133 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 85 85 85 136 136 136 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 F F F 12 240-259 84 84 84 135 135 135 F F F 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 84 84 84 134 134 134 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 82 82 82 130 130 130 F F F 19 380-399 82 82 82 130 130 130 F F F 20 400-419 83 83 83 133 133 133 F F F 21 420-439 84 84 84 134 134 134 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 83 83 83 133 133 133 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 83 83 83 133 133 133 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 F F F 28 560-579 72 72 72 119 119 119 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 85, Max flush: 136 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.211844344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.256700385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.25749237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.257543783Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.25756158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.257579947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} {"msg":"Crucible 0cc64998-6e71-4ec3-b40e-a5a3308b6c26 has session id: 00c0a595-fee4-4d04-8662-0c322a3f0659","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258167192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Upstairs opts: Upstairs UUID: 0cc64998-6e71-4ec3-b40e-a5a3308b6c26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258211956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Crucible stats registered with UUID: 0cc64998-6e71-4ec3-b40e-a5a3308b6c26","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258236942Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:52.258252519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258692646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258709343Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} {"msg":"0cc64998-6e71-4ec3-b40e-a5a3308b6c26 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258777453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258793981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258816117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258834405Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258849443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258862991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258877268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258890557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258904475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.258946369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"ds_connection connected from Ok(127.0.0.1:37989)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259015448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"ds_connection connected from Ok(127.0.0.1:56656)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259054893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"ds_connection connected from Ok(127.0.0.1:49958)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259094397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259774588Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259813702Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.259893391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.260359233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.260394758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.260656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.260892755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261010948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[0]R flush_numbers[0..12]: [135, 129, 124, 124, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261128481Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[0]R generation[0..12]: [84, 81, 77, 77, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261149788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261164836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[1]R flush_numbers[0..12]: [135, 129, 124, 124, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261178884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[1]R generation[0..12]: [84, 81, 77, 77, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261192751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261206409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[2]R flush_numbers[0..12]: [135, 129, 124, 124, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261219788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[2]R generation[0..12]: [84, 81, 77, 77, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261232846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261246444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261259122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.26127193Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:13:52.261288248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261307375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"":"downstairs","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261320423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261332031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"0","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261345009Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"1","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261357787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"client":"2","":"downstairs","session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261370526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261382504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"0cc64998-6e71-4ec3-b40e-a5a3308b6c26 is now active with session: 00c0a595-fee4-4d04-8662-0c322a3f0659","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261394902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.26140646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526,"session_id":"00c0a595-fee4-4d04-8662-0c322a3f0659"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:52.261418479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 12288 data: 6 8 7 02/30 Write block 462 len 28672 data: 7 7 6 6 7 5 7 03/30 Read block 550 len 28672 04/30 Read block 278 len 12288 05/30 Write block 39 len 28672 data: 14 13 9 7 7 4 5 06/30 Read block 82 len 20480 07/30 Write block 375 len 32768 data: 6 5 7 7 7 5 7 7 08/30 Read block 175 len 32768 09/30 Write block 404 len 4096 data: 9 10/30 Read block 325 len 20480 11/30 Flush 12/30 Write block 573 len 16384 data: 9 9 9 8 13/30 Read block 78 len 4096 14/30 Flush 15/30 Write block 63 len 32768 data: 11 11 14 11 9 7 10 9 16/30 Read block 100 len 32768 17/30 Read block 374 len 8192 18/30 Read block 63 len 4096 19/30 Write block 315 len 20480 data: 6 5 6 5 4 20/30 Read block 185 len 16384 21/30 Read block 223 len 16384 22/30 Read block 317 len 32768 23/30 Flush 24/30 Write block 252 len 12288 data: 5 5 6 25/30 Read block 206 len 28672 26/30 Read block 437 len 16384 27/30 Write block 547 len 36864 data: 7 9 8 5 6 5 6 5 8 28/30 Write block 464 len 36864 data: 7 7 8 6 8 6 7 5 6 29/30 Write block 435 len 4096 data: 9 30/30 Read block 526 len 4096 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:52.281977297Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 86 86 86 137 137 137 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 86 86 86 139 139 139 F F F 4 080-099 83 83 83 133 133 133 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 85 85 85 136 136 136 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 F F F 12 240-259 84 84 84 135 135 135 T T T 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 86 86 86 137 137 137 F F F 19 380-399 86 86 86 137 137 137 F F F 20 400-419 86 86 86 137 137 137 F F F 21 420-439 86 86 86 137 137 137 T T T 22 440-459 84 84 84 134 134 134 F F F 23 460-479 86 86 86 137 137 137 T T T 24 480-499 80 80 80 126 126 126 F F F 25 500-519 83 83 83 133 133 133 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 77 77 77 124 124 124 T T T 28 560-579 86 86 86 138 138 138 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 86, Max flush: 139 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 86 86 86 137 137 137 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 86 86 86 139 139 139 F F F 4 080-099 83 83 83 133 133 133 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 78 78 78 125 125 125 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 85 85 85 136 136 136 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 F F F 12 240-259 87 87 87 140 140 140 F F F 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 86 86 86 137 137 137 F F F 19 380-399 86 86 86 137 137 137 F F F 20 400-419 86 86 86 137 137 137 F F F 21 420-439 87 87 87 140 140 140 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 87 87 87 140 140 140 F F F 24 480-499 80 80 80 126 126 126 F F F 25 500-519 83 83 83 133 133 133 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 87 87 87 140 140 140 F F F 28 560-579 86 86 86 138 138 138 F F F 29 580-599 75 75 75 122 122 122 F F F Max gen: 87, Max flush: 140 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.516541758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:56.559754989Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.560120566Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.592089683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.592656061Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.592691216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.592705643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.592719781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} {"msg":"Crucible cf795255-ce30-463e-bfd8-9563b07be320 has session id: 9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593292718Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Upstairs opts: Upstairs UUID: cf795255-ce30-463e-bfd8-9563b07be320, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593333672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Crucible stats registered with UUID: cf795255-ce30-463e-bfd8-9563b07be320","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593361448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:56.593382235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593734784Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593767679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} {"msg":"cf795255-ce30-463e-bfd8-9563b07be320 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593822561Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593849137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593873923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593888691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593913358Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.593976758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594002845Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594025101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594052677Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594079873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61012)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594108489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57485)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594251838Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52711)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594296392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594726999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594768623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.594982882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.595470281Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.595498697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.595522213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596040438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596076103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[0]R flush_numbers[0..12]: [135, 137, 137, 139, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.59609439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[0]R generation[0..12]: [84, 86, 86, 86, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596109488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596123826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[1]R flush_numbers[0..12]: [135, 137, 137, 139, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596137144Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[1]R generation[0..12]: [84, 86, 86, 86, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596150482Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596167979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[2]R flush_numbers[0..12]: [135, 137, 137, 139, 133, 119, 133, 125, 134, 136, 124, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596181888Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[2]R generation[0..12]: [84, 86, 86, 86, 83, 72, 83, 78, 84, 85, 77, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596195106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596208684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596221352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.5962344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596247028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596259666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"":"downstairs","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:13:56.596272234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596295371Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"0","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596319257Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"1","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596342254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"client":"2","":"downstairs","session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596362691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596384738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"cf795255-ce30-463e-bfd8-9563b07be320 is now active with session: 9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596405595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596420563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534,"session_id":"9c17fc3c-30d6-4f6d-a3a8-0b3871f17edc"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:13:56.596432861Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 12288 data: 8 8 8 02/30 Write block 521 len 8192 data: 8 8 03/30 Write block 300 len 20480 data: 7 7 6 6 8 04/30 Write block 376 len 20480 data: 6 8 8 8 6 05/30 Write block 552 len 36864 data: 6 7 6 9 9 7 8 7 5 06/30 Write block 146 len 16384 data: 9 10 11 14 07/30 Write block 589 len 32768 data: 7 6 5 4 4 4 4 5 08/30 Read block 136 len 20480 09/30 Write block 430 len 40960 data: 9 7 5 5 5 10 8 7 6 5 10/30 Read block 145 len 16384 11/30 Read block 283 len 4096 12/30 Flush 13/30 Write block 75 len 36864 data: 7 6 7 6 7 6 4 4 5 14/30 Read block 501 len 16384 15/30 Flush 16/30 Write block 577 len 24576 data: 9 7 6 5 7 7 17/30 Write block 500 len 20480 data: 7 8 7 10 10 18/30 Read block 253 len 24576 19/30 Write block 472 len 12288 data: 7 6 5 20/30 Write block 479 len 28672 data: 6 6 6 6 5 5 5 21/30 Write block 184 len 24576 data: 11 9 9 9 8 9 22/30 Write block 415 len 28672 data: 7 7 7 8 9 9 8 23/30 Read block 281 len 8192 24/30 Read block 199 len 32768 25/30 Read block 400 len 16384 26/30 Read block 498 len 8192 27/30 Read block 61 len 24576 28/30 Read block 40 len 40960 29/30 Write block 235 len 36864 data: 9 11 8 8 9 10 7 10 6 30/30 Write block 566 len 12288 data: 5 6 7 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:13:56.614387458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 86 86 86 137 137 137 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 88 88 88 141 141 141 T T T 10 200-219 77 77 77 124 124 124 F F F 11 220-239 83 83 83 133 133 133 T T T 12 240-259 87 87 87 140 140 140 T T T 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 88 88 88 141 141 141 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 88 88 88 141 141 141 F F F 20 400-419 86 86 86 137 137 137 T T T 21 420-439 88 88 88 141 141 141 T T T 22 440-459 84 84 84 134 134 134 F F F 23 460-479 87 87 87 140 140 140 T T T 24 480-499 80 80 80 126 126 126 T T T 25 500-519 83 83 83 133 133 133 T T T 26 520-539 88 88 88 141 141 141 F F F 27 540-559 88 88 88 141 141 141 F F F 28 560-579 88 88 88 141 141 141 T T T 29 580-599 88 88 88 141 141 141 T T T Max gen: 88, Max flush: 142 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 86 86 86 137 137 137 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 89 89 89 143 143 143 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 89 89 89 143 143 143 F F F 12 240-259 89 89 89 143 143 143 F F F 13 260-279 83 83 83 133 133 133 F F F 14 280-299 83 83 83 133 133 133 F F F 15 300-319 88 88 88 141 141 141 F F F 16 320-339 78 78 78 125 125 125 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 88 88 88 141 141 141 F F F 20 400-419 89 89 89 143 143 143 F F F 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 89 89 89 143 143 143 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 88 88 88 141 141 141 F F F 27 540-559 88 88 88 141 141 141 F F F 28 560-579 89 89 89 143 143 143 F F F 29 580-599 89 89 89 143 143 143 F F F Max gen: 89, Max flush: 143 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.908984651Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.952729144Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953300271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953357573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.95337555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953391268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} {"msg":"Crucible 2e4d3200-152e-4954-a82e-e8909e754d00 has session id: 268353fa-1d43-480c-b845-1f58996ec813","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953894205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Upstairs opts: Upstairs UUID: 2e4d3200-152e-4954-a82e-e8909e754d00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953944527Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Crucible stats registered with UUID: 2e4d3200-152e-4954-a82e-e8909e754d00","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.953966374Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:00.953980402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954415449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954440665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} {"msg":"2e4d3200-152e-4954-a82e-e8909e754d00 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954499287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954525463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954549659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954564457Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954579255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954596432Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954618209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954635257Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954649495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954668322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"ds_connection connected from Ok(127.0.0.1:36868)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954766458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"ds_connection connected from Ok(127.0.0.1:45556)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954804322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"ds_connection connected from Ok(127.0.0.1:60126)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.954834538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.955481354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.955511849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956150556Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956190571Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956215487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956722543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956762597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.956787793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[0]R flush_numbers[0..12]: [135, 137, 137, 142, 142, 119, 133, 141, 134, 143, 124, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957258865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[0]R generation[0..12]: [84, 86, 86, 88, 88, 72, 83, 88, 84, 89, 77, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957285971Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957301699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[1]R flush_numbers[0..12]: [135, 137, 137, 142, 142, 119, 133, 141, 134, 143, 124, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957315777Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[1]R generation[0..12]: [84, 86, 86, 88, 88, 72, 83, 88, 84, 89, 77, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957329155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957343083Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[2]R flush_numbers[0..12]: [135, 137, 137, 142, 142, 119, 133, 141, 134, 143, 124, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957375218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[2]R generation[0..12]: [84, 86, 86, 88, 88, 72, 83, 88, 84, 89, 77, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957388976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957402724Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957415542Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.95743439Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957447118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957463065Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"":"downstairs","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957476293Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957488122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"0","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.95750097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"1","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957513638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"client":"2","":"downstairs","session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957526876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957539454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"2e4d3200-152e-4954-a82e-e8909e754d00 is now active with session: 268353fa-1d43-480c-b845-1f58996ec813","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957551942Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957563591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542,"session_id":"268353fa-1d43-480c-b845-1f58996ec813"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:00.957578629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 281 len 28672 data: 9 8 7 10 12 11 12 02/30 Read block 69 len 28672 03/30 Write block 276 len 12288 data: 9 10 9 04/30 Write block 30 len 20480 data: 6 7 11 9 9 05/30 Read block 24 len 20480 06/30 Write block 234 len 40960 data: 9 10 12 9 9 10 11 8 11 7 07/30 Read block 306 len 20480 08/30 Read block 559 len 32768 09/30 Write block 414 len 12288 data: 8 8 8 10/30 Write block 581 len 24576 data: 8 8 5 9 10 10 11/30 Write block 339 len 16384 data: 11 9 6 9 12/30 Read block 130 len 12288 13/30 Write block 272 len 28672 data: 8 9 10 11 10 11 10 14/30 Read block 551 len 32768 15/30 Write block 525 len 12288 data: 8 8 8 16/30 Read block 13 len 32768 17/30 Read block 335 len 28672 18/30 Write block 283 len 16384 data: 8 11 13 12 19/30 Read block 148 len 12288 20/30 Read block 280 len 36864 21/30 Write block 547 len 16384 data: 8 10 9 6 22/30 Write block 132 len 24576 data: 6 6 8 5 5 5 23/30 Read block 254 len 24576 24/30 Write block 256 len 28672 data: 8 8 7 7 6 5 5 25/30 Read block 20 len 32768 26/30 Read block 568 len 32768 27/30 Read block 70 len 20480 28/30 Read block 551 len 28672 29/30 Read block 63 len 20480 30/30 Read block 461 len 24576 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:00.978142136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 86 86 86 137 137 137 T T T 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 83 83 83 133 133 133 T T T 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 89 89 89 143 143 143 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 89 89 89 143 143 143 T T T 12 240-259 89 89 89 143 143 143 T T T 13 260-279 83 83 83 133 133 133 T T T 14 280-299 83 83 83 133 133 133 T T T 15 300-319 88 88 88 141 141 141 F F F 16 320-339 78 78 78 125 125 125 T T T 17 340-359 82 82 82 132 132 132 T T T 18 360-379 88 88 88 141 141 141 F F F 19 380-399 88 88 88 141 141 141 F F F 20 400-419 89 89 89 143 143 143 T T T 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 89 89 89 143 143 143 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 88 88 88 141 141 141 T T T 27 540-559 88 88 88 141 141 141 T T T 28 560-579 89 89 89 143 143 143 F F F 29 580-599 89 89 89 143 143 143 T T T Max gen: 89, Max flush: 143 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 F F F 1 020-039 90 90 90 144 144 144 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 89 89 89 143 143 143 F F F 10 200-219 77 77 77 124 124 124 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 90 90 90 144 144 144 F F F 13 260-279 90 90 90 144 144 144 F F F 14 280-299 90 90 90 144 144 144 F F F 15 300-319 88 88 88 141 141 141 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 90 90 90 144 144 144 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 88 88 88 141 141 141 F F F 20 400-419 90 90 90 144 144 144 F F F 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 89 89 89 143 143 143 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 90 90 90 144 144 144 F F F 28 560-579 89 89 89 143 143 143 F F F 29 580-599 90 90 90 144 144 144 F F F Max gen: 90, Max flush: 144 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.259346609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.303266897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.303929991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.303970745Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.303985063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30400387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} {"msg":"Crucible 39a03fd0-002d-4fc3-89a4-95c2824c5e0e has session id: 5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.304661974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Upstairs opts: Upstairs UUID: 39a03fd0-002d-4fc3-89a4-95c2824c5e0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.304699189Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Crucible stats registered with UUID: 39a03fd0-002d-4fc3-89a4-95c2824c5e0e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.304723855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:05.304748032Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305164781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305189138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} {"msg":"39a03fd0-002d-4fc3-89a4-95c2824c5e0e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305268026Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305285313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30530696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305320508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305334296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305348444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305372581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305396697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305418844Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30544458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42879)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305482055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48722)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305609216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52217)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.305648031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.306183493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.306213658Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.306284698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.306950681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.306979387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307065494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307542255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307691473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[0]R flush_numbers[0..12]: [135, 144, 137, 142, 142, 119, 144, 141, 134, 143, 124, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307743506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[0]R generation[0..12]: [84, 90, 86, 88, 88, 72, 90, 88, 84, 89, 77, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307763183Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307778361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[1]R flush_numbers[0..12]: [135, 144, 137, 142, 142, 119, 144, 141, 134, 143, 124, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307797018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[1]R generation[0..12]: [84, 90, 86, 88, 88, 72, 90, 88, 84, 89, 77, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307811186Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307825124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[2]R flush_numbers[0..12]: [135, 144, 137, 142, 142, 119, 144, 141, 134, 143, 124, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307838692Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[2]R generation[0..12]: [84, 90, 86, 88, 88, 72, 90, 88, 84, 89, 77, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30785203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307865708Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Max found gen is 91","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307878336Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Generation requested: 92 >= found:91","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307891154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307903672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30791633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"":"downstairs","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-14T16:14:05.307936558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307949496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"0","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307962434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"1","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.307975232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"client":"2","":"downstairs","session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.30798813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.308002598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"39a03fd0-002d-4fc3-89a4-95c2824c5e0e is now active with session: 5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.308015606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.308027584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550,"session_id":"5f286dc7-df6b-47b1-9bbe-5b0dbc8c8fa4"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:05.308039423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 16384 data: 7 9 10 10 02/30 Write block 31 len 32768 data: 8 12 10 10 8 8 9 11 03/30 Read block 557 len 20480 04/30 Read block 432 len 4096 05/30 Flush 06/30 Write block 309 len 36864 data: 6 6 6 7 8 7 7 6 7 07/30 Flush 08/30 Write block 18 len 12288 data: 4 4 4 09/30 Write block 399 len 20480 data: 12 11 9 8 10 10/30 Write block 471 len 32768 data: 6 8 7 6 7 7 7 7 11/30 Read block 209 len 40960 12/30 Read block 289 len 4096 13/30 Write block 195 len 24576 data: 7 8 9 10 8 7 14/30 Read block 359 len 24576 15/30 Write block 526 len 20480 data: 9 9 9 8 10 16/30 Read block 485 len 4096 17/30 Read block 256 len 28672 18/30 Write block 403 len 32768 data: 11 11 11 10 9 10 10 11 19/30 Write block 403 len 28672 data: 12 12 12 11 10 11 11 20/30 Write block 87 len 40960 data: 4 5 6 5 4 5 4 4 8 9 21/30 Write block 20 len 4096 data: 5 22/30 Write block 385 len 24576 data: 6 5 6 8 10 11 23/30 Flush 24/30 Write block 312 len 4096 data: 8 25/30 Read block 28 len 36864 26/30 Read block 149 len 40960 27/30 Write block 197 len 12288 data: 10 11 9 28/30 Write block 272 len 4096 data: 9 29/30 Write block 317 len 28672 data: 8 6 5 3 3 4 5 30/30 Read block 98 len 8192 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 7 5 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:05.326182712Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 147 147 147 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 92 92 92 147 147 147 T T T 10 200-219 92 92 92 147 147 147 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 90 90 90 144 144 144 F F F 13 260-279 90 90 90 144 144 144 T T T 14 280-299 90 90 90 144 144 144 F F F 15 300-319 92 92 92 146 146 146 T T T 16 320-339 90 90 90 144 144 144 T T T 17 340-359 90 90 90 144 144 144 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 92 92 92 147 147 147 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 92 92 92 147 147 147 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 90 90 90 144 144 144 F F F 28 560-579 89 89 89 143 143 143 F F F 29 580-599 90 90 90 144 144 144 F F F Max gen: 92, Max flush: 147 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 147 147 147 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 86 86 86 137 137 137 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 93 93 93 148 148 148 F F F 10 200-219 92 92 92 147 147 147 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 90 90 90 144 144 144 F F F 13 260-279 93 93 93 148 148 148 F F F 14 280-299 90 90 90 144 144 144 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 93 93 93 148 148 148 F F F 17 340-359 90 90 90 144 144 144 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 92 92 92 147 147 147 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 92 92 92 147 147 147 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 90 90 90 144 144 144 F F F 28 560-579 89 89 89 143 143 143 F F F 29 580-599 90 90 90 144 144 144 F F F Max gen: 93, Max flush: 148 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.556477515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.600719666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601345645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601395497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601410925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601425433Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} {"msg":"Crucible 7efb7b4c-91ed-45ad-b184-c2f3be0c61aa has session id: fe58c9f9-7503-4629-9285-7029f0f8e05d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601955036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Upstairs opts: Upstairs UUID: 7efb7b4c-91ed-45ad-b184-c2f3be0c61aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.601999489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Crucible stats registered with UUID: 7efb7b4c-91ed-45ad-b184-c2f3be0c61aa","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602023356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:09.602044513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602518964Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602542461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} {"msg":"7efb7b4c-91ed-45ad-b184-c2f3be0c61aa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602598962Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.60261579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602638716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602652514Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602666592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602689949Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602706687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602720215Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602733973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602747311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59759)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602876092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40439)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.602915706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64698)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.603081482Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.603554633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.603582729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.603675736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.60426583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604297375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604325621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604833197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604868572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[0]R flush_numbers[0..12]: [147, 147, 137, 142, 147, 119, 144, 141, 134, 148, 147, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604892319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[0]R generation[0..12]: [92, 92, 86, 88, 92, 72, 90, 88, 84, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604908696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604922784Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[1]R flush_numbers[0..12]: [147, 147, 137, 142, 147, 119, 144, 141, 134, 148, 147, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604936422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[1]R generation[0..12]: [92, 92, 86, 88, 92, 72, 90, 88, 84, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.60494986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604963688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[2]R flush_numbers[0..12]: [147, 147, 137, 142, 147, 119, 144, 141, 134, 148, 147, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604977406Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[2]R generation[0..12]: [92, 92, 86, 88, 92, 72, 90, 88, 84, 93, 92, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.604990794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605004752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.6050178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605031218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605053265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605085411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"":"downstairs","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605106697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605119426Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"0","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605132504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"1","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605145422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"client":"2","":"downstairs","session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.60515834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605170668Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"7efb7b4c-91ed-45ad-b184-c2f3be0c61aa is now active with session: fe58c9f9-7503-4629-9285-7029f0f8e05d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605183296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605195115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558,"session_id":"fe58c9f9-7503-4629-9285-7029f0f8e05d"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:09.605207013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 32768 data: 9 8 7 8 8 8 8 7 02/30 Read block 495 len 12288 03/30 Write block 467 len 36864 data: 7 9 7 8 7 10 9 8 9 04/30 Flush 05/30 Write block 44 len 16384 data: 5 6 7 8 06/30 Read block 341 len 36864 07/30 Write block 296 len 8192 data: 12 9 08/30 Read block 471 len 8192 09/30 Write block 401 len 28672 data: 10 9 13 13 13 12 11 10/30 Write block 542 len 12288 data: 9 8 7 11/30 Read block 464 len 4096 12/30 Read block 377 len 8192 13/30 Read block 68 len 12288 14/30 Write block 194 len 32768 data: 8 8 9 11 12 10 8 7 15/30 Read block 227 len 8192 16/30 Write block 375 len 24576 data: 7 7 9 9 9 7 17/30 Write block 414 len 32768 data: 9 9 9 8 9 10 10 9 18/30 Write block 4 len 16384 data: 6 5 4 5 19/30 Read block 338 len 8192 20/30 Read block 292 len 36864 21/30 Read block 87 len 4096 22/30 Write block 207 len 20480 data: 6 9 9 8 8 23/30 Read block 581 len 32768 24/30 Flush 25/30 Write block 594 len 20480 data: 5 5 6 5 5 26/30 Write block 349 len 32768 data: 10 9 9 9 10 11 11 12 27/30 Read block 163 len 40960 28/30 Read block 146 len 28672 29/30 Write block 241 len 32768 data: 9 12 8 6 7 6 6 6 30/30 Write block 567 len 20480 data: 7 8 8 8 8 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:09.624198639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 150 150 150 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 94 94 94 150 150 150 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 90 90 90 144 144 144 T T T 13 260-279 93 93 93 148 148 148 F F F 14 280-299 94 94 94 150 150 150 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 93 93 93 148 148 148 F F F 17 340-359 90 90 90 144 144 144 T T T 18 360-379 94 94 94 150 150 150 F F F 19 380-399 94 94 94 150 150 150 F F F 20 400-419 94 94 94 150 150 150 F F F 21 420-439 94 94 94 150 150 150 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 94 94 94 149 149 149 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 89 89 89 143 143 143 T T T 29 580-599 90 90 90 144 144 144 T T T Max gen: 94, Max flush: 150 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 150 150 150 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 94 94 94 150 150 150 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 84 84 84 134 134 134 F F F 9 180-199 94 94 94 150 150 150 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 93 93 93 148 148 148 F F F 14 280-299 94 94 94 150 150 150 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 93 93 93 148 148 148 F F F 17 340-359 95 95 95 151 151 151 F F F 18 360-379 94 94 94 150 150 150 F F F 19 380-399 94 94 94 150 150 150 F F F 20 400-419 94 94 94 150 150 150 F F F 21 420-439 94 94 94 150 150 150 F F F 22 440-459 84 84 84 134 134 134 F F F 23 460-479 94 94 94 149 149 149 F F F 24 480-499 89 89 89 143 143 143 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 95, Max flush: 151 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.857553906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.901161749Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.901746074Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.901787808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.901803606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.901817824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} {"msg":"Crucible 29e4b0ed-3e34-4e3b-a573-5431a8265cb5 has session id: e8e7c634-4e3f-4f0d-9054-e7d67641434b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902356626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Upstairs opts: Upstairs UUID: 29e4b0ed-3e34-4e3b-a573-5431a8265cb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.90239177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Crucible stats registered with UUID: 29e4b0ed-3e34-4e3b-a573-5431a8265cb5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902407388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:13.902420266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902867961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902898736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} {"msg":"29e4b0ed-3e34-4e3b-a573-5431a8265cb5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902957368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902974945Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.902998312Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.90301223Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903026378Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903040426Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903064443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903087489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903104866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903118455Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47797)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903317775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55559)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903352641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60275)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.903377947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.904065217Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.90410925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.904551506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.904725271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.904860821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905214899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905338621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905462263Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[0]R flush_numbers[0..12]: [150, 147, 150, 142, 147, 119, 144, 141, 134, 150, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905834859Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[0]R generation[0..12]: [94, 92, 94, 88, 92, 72, 90, 88, 84, 94, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905861505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905877273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[1]R flush_numbers[0..12]: [150, 147, 150, 142, 147, 119, 144, 141, 134, 150, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905891451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[1]R generation[0..12]: [94, 92, 94, 88, 92, 72, 90, 88, 84, 94, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905904789Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905918407Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[2]R flush_numbers[0..12]: [150, 147, 150, 142, 147, 119, 144, 141, 134, 150, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905931805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[2]R generation[0..12]: [94, 92, 94, 88, 92, 72, 90, 88, 84, 94, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905948583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905962671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905975489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.905988277Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906015723Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.90603766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"":"downstairs","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906050988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906062606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"0","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906075614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"1","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906088252Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"client":"2","":"downstairs","session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906101141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906113019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"29e4b0ed-3e34-4e3b-a573-5431a8265cb5 is now active with session: e8e7c634-4e3f-4f0d-9054-e7d67641434b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906125317Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906136885Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566,"session_id":"e8e7c634-4e3f-4f0d-9054-e7d67641434b"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:13.906148714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 426 len 16384 data: 7 7 9 8 02/30 Read block 383 len 4096 03/30 Write block 496 len 16384 data: 5 6 5 7 04/30 Write block 265 len 12288 data: 7 9 7 05/30 Write block 53 len 24576 data: 8 8 7 6 8 7 06/30 Read block 67 len 16384 07/30 Write block 380 len 16384 data: 8 8 8 7 08/30 Flush 09/30 Flush 10/30 Write block 429 len 24576 data: 9 10 8 6 6 6 11/30 Write block 440 len 24576 data: 5 6 7 8 7 6 12/30 Write block 330 len 36864 data: 11 11 10 8 8 10 9 11 12 13/30 Read block 392 len 16384 14/30 Read block 497 len 16384 15/30 Read block 74 len 16384 16/30 Read block 507 len 16384 17/30 Write block 281 len 20480 data: 10 9 9 12 14 18/30 Flush 19/30 Write block 499 len 40960 data: 8 8 9 8 11 11 11 12 14 13 20/30 Flush 21/30 Write block 163 len 20480 data: 6 7 8 8 7 22/30 Read block 530 len 8192 23/30 Flush 24/30 Write block 189 len 16384 data: 10 9 9 7 25/30 Write block 321 len 28672 data: 4 5 6 8 9 9 10 26/30 Read block 259 len 32768 27/30 Write block 395 len 16384 data: 10 10 12 13 28/30 Read block 91 len 20480 29/30 Read block 522 len 16384 30/30 Read block 540 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:13.925162096Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 150 150 150 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 96 96 96 152 152 152 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 96 96 96 156 156 156 F F F 9 180-199 94 94 94 150 150 150 T T T 10 200-219 94 94 94 150 150 150 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 96 96 96 154 154 154 T T T 17 340-359 95 95 95 151 151 151 F F F 18 360-379 94 94 94 150 150 150 F F F 19 380-399 96 96 96 152 152 152 T T T 20 400-419 94 94 94 150 150 150 F F F 21 420-439 96 96 96 154 154 154 F F F 22 440-459 96 96 96 154 154 154 F F F 23 460-479 94 94 94 149 149 149 F F F 24 480-499 96 96 96 155 155 155 F F F 25 500-519 96 96 96 155 155 155 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 96, Max flush: 156 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 150 150 150 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 96 96 96 152 152 152 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 F F F 5 100-119 72 72 72 119 119 119 F F F 6 120-139 90 90 90 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 96 96 96 156 156 156 F F F 9 180-199 97 97 97 157 157 157 F F F 10 200-219 94 94 94 150 150 150 F F F 11 220-239 90 90 90 144 144 144 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 97 97 97 157 157 157 F F F 17 340-359 95 95 95 151 151 151 F F F 18 360-379 94 94 94 150 150 150 F F F 19 380-399 97 97 97 157 157 157 F F F 20 400-419 94 94 94 150 150 150 F F F 21 420-439 96 96 96 154 154 154 F F F 22 440-459 96 96 96 154 154 154 F F F 23 460-479 94 94 94 149 149 149 F F F 24 480-499 96 96 96 155 155 155 F F F 25 500-519 96 96 96 155 155 155 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 97, Max flush: 157 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.153490886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:18.196964829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.197352822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.229791661Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.230425848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.230460014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.230474261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.230492999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} {"msg":"Crucible e32d2316-194f-42d4-9476-58806080d87a has session id: 3841a54f-b1e0-419c-9032-f80c684aa1ed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231120018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Upstairs opts: Upstairs UUID: e32d2316-194f-42d4-9476-58806080d87a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231166441Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Crucible stats registered with UUID: e32d2316-194f-42d4-9476-58806080d87a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231192307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:18.231206755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.23165656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231690455Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} {"msg":"e32d2316-194f-42d4-9476-58806080d87a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231743337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231769803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231793429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231807807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231822075Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231836063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.23186103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231886696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231919821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231948437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:57665)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.231976033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:40882)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.232005109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:48170)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.232060681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.232658014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.2326853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.232700148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.23323317Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.233264935Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.233390617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.233794178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.233894964Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[0]R flush_numbers[0..12]: [150, 147, 152, 142, 147, 119, 144, 141, 156, 157, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234004448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[0]R generation[0..12]: [94, 92, 96, 88, 92, 72, 90, 88, 96, 97, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234026695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234041112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[1]R flush_numbers[0..12]: [150, 147, 152, 142, 147, 119, 144, 141, 156, 157, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.23405466Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[1]R generation[0..12]: [94, 92, 96, 88, 92, 72, 90, 88, 96, 97, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234068038Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234081756Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[2]R flush_numbers[0..12]: [150, 147, 152, 142, 147, 119, 144, 141, 156, 157, 150, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234095325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[2]R generation[0..12]: [94, 92, 96, 88, 92, 72, 90, 88, 96, 97, 94, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234108583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234122031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234134759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234147797Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234160385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:14:18.234172823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"":"downstairs","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234188621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234200409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"0","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234213467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"1","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234228135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"client":"2","":"downstairs","session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234241253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234253242Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"e32d2316-194f-42d4-9476-58806080d87a is now active with session: 3841a54f-b1e0-419c-9032-f80c684aa1ed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.2342658Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234277528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574,"session_id":"3841a54f-b1e0-419c-9032-f80c684aa1ed"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:18.234289536Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 24576 data: 8 9 6 5 6 6 02/30 Read block 533 len 12288 03/30 Flush 04/30 Write block 431 len 40960 data: 9 7 7 7 11 9 8 7 6 6 05/30 Read block 84 len 16384 06/30 Read block 12 len 8192 07/30 Write block 93 len 36864 data: 5 5 9 10 8 9 11 11 11 08/30 Read block 576 len 16384 09/30 Write block 10 len 12288 data: 5 4 5 10/30 Read block 222 len 24576 11/30 Read block 211 len 12288 12/30 Read block 447 len 24576 13/30 Write block 143 len 28672 data: 9 10 12 10 11 12 15 14/30 Write block 402 len 36864 data: 10 14 14 14 13 12 12 12 12 15/30 Read block 487 len 4096 16/30 Write block 507 len 8192 data: 15 14 17/30 Read block 395 len 8192 18/30 Read block 139 len 24576 19/30 Write block 443 len 16384 data: 9 8 7 7 20/30 Write block 497 len 36864 data: 7 6 9 9 10 9 12 12 12 21/30 Write block 229 len 28672 data: 4 4 5 9 7 10 11 22/30 Write block 186 len 28672 data: 10 10 9 11 10 10 8 23/30 Write block 441 len 20480 data: 7 8 10 9 8 24/30 Read block 360 len 8192 25/30 Write block 127 len 28672 data: 3 4 5 6 6 7 7 26/30 Read block 183 len 16384 27/30 Write block 475 len 20480 data: 10 9 9 9 8 28/30 Write block 323 len 12288 data: 7 9 10 29/30 Read block 576 len 28672 30/30 Read block 432 len 28672 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:18.253700791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 150 150 150 T T T 1 020-039 92 92 92 147 147 147 F F F 2 040-059 96 96 96 152 152 152 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 92 92 92 147 147 147 T T T 5 100-119 72 72 72 119 119 119 T T T 6 120-139 90 90 90 144 144 144 T T T 7 140-159 88 88 88 141 141 141 T T T 8 160-179 96 96 96 156 156 156 F F F 9 180-199 97 97 97 157 157 157 T T T 10 200-219 98 98 98 158 158 158 F F F 11 220-239 90 90 90 144 144 144 T T T 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 97 97 97 157 157 157 T T T 17 340-359 95 95 95 151 151 151 F F F 18 360-379 94 94 94 150 150 150 F F F 19 380-399 97 97 97 157 157 157 F F F 20 400-419 94 94 94 150 150 150 T T T 21 420-439 96 96 96 154 154 154 T T T 22 440-459 96 96 96 154 154 154 T T T 23 460-479 94 94 94 149 149 149 T T T 24 480-499 96 96 96 155 155 155 T T T 25 500-519 96 96 96 155 155 155 T T T 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 98, Max flush: 158 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 159 159 159 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 96 96 96 152 152 152 F F F 3 060-079 88 88 88 142 142 142 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 99 99 99 159 159 159 F F F 6 120-139 99 99 99 159 159 159 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 96 96 96 156 156 156 F F F 9 180-199 99 99 99 159 159 159 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 99 99 99 159 159 159 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 93 93 93 148 148 148 F F F 16 320-339 99 99 99 159 159 159 F F F 17 340-359 95 95 95 151 151 151 F F F 18 360-379 94 94 94 150 150 150 F F F 19 380-399 97 97 97 157 157 157 F F F 20 400-419 99 99 99 159 159 159 F F F 21 420-439 99 99 99 159 159 159 F F F 22 440-459 99 99 99 159 159 159 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 99 99 99 159 159 159 F F F 25 500-519 99 99 99 159 159 159 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 99, Max flush: 159 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.563416915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:22.605870126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.606232533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.63772657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.638288478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.638324023Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.638338111Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.638353179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} {"msg":"Crucible 52912847-7797-4265-8731-2400646947aa has session id: 8eebfdfd-a657-4dc7-8ff8-2f67d9e19488","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.638988136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Upstairs opts: Upstairs UUID: 52912847-7797-4265-8731-2400646947aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.63903085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Crucible stats registered with UUID: 52912847-7797-4265-8731-2400646947aa","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639058436Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:22.639079673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639531737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639559413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} {"msg":"52912847-7797-4265-8731-2400646947aa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639613005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639640391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639677106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639707552Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639739547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639767693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639798028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639822635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639846061Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639873377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"ds_connection connected from Ok(127.0.0.1:45034)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.639900084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"ds_connection connected from Ok(127.0.0.1:44538)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.640008598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"ds_connection connected from Ok(127.0.0.1:33018)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.640037494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.640495587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.640535051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.6406142Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641108058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641147772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641248327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641616034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641701431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[0]R flush_numbers[0..12]: [159, 147, 152, 142, 159, 159, 159, 159, 156, 159, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641804776Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[0]R generation[0..12]: [99, 92, 96, 88, 99, 99, 99, 99, 96, 99, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641823754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641838122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[1]R flush_numbers[0..12]: [159, 147, 152, 142, 159, 159, 159, 159, 156, 159, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.64185196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[1]R generation[0..12]: [99, 92, 96, 88, 99, 99, 99, 99, 96, 99, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641865158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641883175Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[2]R flush_numbers[0..12]: [159, 147, 152, 142, 159, 159, 159, 159, 156, 159, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641897263Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[2]R generation[0..12]: [99, 92, 96, 88, 99, 99, 99, 99, 96, 99, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641910641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641924449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641937227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641950135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641962664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.641975092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"":"downstairs","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:14:22.64198773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642017085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"0","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642031483Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"1","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642044332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"client":"2","":"downstairs","session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.6420572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642069198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"52912847-7797-4265-8731-2400646947aa is now active with session: 8eebfdfd-a657-4dc7-8ff8-2f67d9e19488","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642083876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642106912Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582,"session_id":"8eebfdfd-a657-4dc7-8ff8-2f67d9e19488"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:22.642120441Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 366 len 20480 data: 8 8 7 6 7 02/30 Read block 16 len 36864 03/30 Read block 407 len 40960 04/30 Flush 05/30 Write block 45 len 12288 data: 7 8 9 06/30 Read block 188 len 40960 07/30 Read block 577 len 8192 08/30 Write block 61 len 12288 data: 4 8 12 09/30 Read block 282 len 8192 10/30 Write block 480 len 28672 data: 7 7 7 6 6 6 6 11/30 Flush 12/30 Write block 121 len 16384 data: 5 4 5 4 13/30 Write block 344 len 24576 data: 8 10 10 9 9 11 14/30 Read block 284 len 4096 15/30 Read block 217 len 12288 16/30 Read block 224 len 12288 17/30 Flush 18/30 Flush 19/30 Write block 446 len 24576 data: 8 9 7 7 6 7 20/30 Read block 236 len 24576 21/30 Flush 22/30 Write block 185 len 36864 data: 10 11 11 10 12 11 11 9 8 23/30 Write block 495 len 32768 data: 4 6 8 7 10 10 11 10 24/30 Flush 25/30 Write block 308 len 8192 data: 6 7 26/30 Read block 251 len 24576 27/30 Read block 534 len 16384 28/30 Read block 479 len 16384 29/30 Read block 62 len 28672 30/30 Write block 574 len 8192 data: 10 10 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:22.661266314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 159 159 159 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 100 100 100 161 161 161 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 99 99 99 159 159 159 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 96 96 96 156 156 156 F F F 9 180-199 100 100 100 165 165 165 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 99 99 99 159 159 159 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 93 93 93 148 148 148 T T T 16 320-339 99 99 99 159 159 159 F F F 17 340-359 100 100 100 162 162 162 F F F 18 360-379 100 100 100 160 160 160 F F F 19 380-399 97 97 97 157 157 157 F F F 20 400-419 99 99 99 159 159 159 F F F 21 420-439 99 99 99 159 159 159 F F F 22 440-459 100 100 100 164 164 164 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 100 100 100 165 165 165 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 95 95 95 151 151 151 T T T 29 580-599 95 95 95 151 151 151 F F F Max gen: 100, Max flush: 165 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 159 159 159 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 100 100 100 161 161 161 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 99 99 99 159 159 159 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 96 96 96 156 156 156 F F F 9 180-199 100 100 100 165 165 165 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 99 99 99 159 159 159 F F F 12 240-259 95 95 95 151 151 151 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 99 99 99 159 159 159 F F F 17 340-359 100 100 100 162 162 162 F F F 18 360-379 100 100 100 160 160 160 F F F 19 380-399 97 97 97 157 157 157 F F F 20 400-419 99 99 99 159 159 159 F F F 21 420-439 99 99 99 159 159 159 F F F 22 440-459 100 100 100 164 164 164 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 100 100 100 165 165 165 F F F 26 520-539 92 92 92 147 147 147 F F F 27 540-559 94 94 94 150 150 150 F F F 28 560-579 101 101 101 166 166 166 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 101, Max flush: 166 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.915505672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:26.959025258Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.959399824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.991628553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99219669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992230415Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992248153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992263141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} {"msg":"Crucible c702a541-f71b-4221-b79c-e7cf4b865fe8 has session id: 00ce8a0b-7cdd-4e89-9a6d-b8653df520f6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992875261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Upstairs opts: Upstairs UUID: c702a541-f71b-4221-b79c-e7cf4b865fe8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992922205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Crucible stats registered with UUID: c702a541-f71b-4221-b79c-e7cf4b865fe8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.992941162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:26.99295633Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993399825Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993422752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} {"msg":"c702a541-f71b-4221-b79c-e7cf4b865fe8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993478674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99350603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993531836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993557112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993584328Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993611734Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993645649Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993674075Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993700291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993723628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41604)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993748034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45066)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993775521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:52267)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.993802266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99446736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.994502125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.994529611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995093489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995120845Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995135652Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995639989Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995755802Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[0]R flush_numbers[0..12]: [159, 147, 161, 161, 159, 159, 162, 159, 156, 165, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995793187Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[0]R generation[0..12]: [99, 92, 100, 100, 99, 99, 100, 99, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995809465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995824222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[1]R flush_numbers[0..12]: [159, 147, 161, 161, 159, 159, 162, 159, 156, 165, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99583819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[1]R generation[0..12]: [99, 92, 100, 100, 99, 99, 100, 99, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995851808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995865646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[2]R flush_numbers[0..12]: [159, 147, 161, 161, 159, 159, 162, 159, 156, 165, 158, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995879024Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[2]R generation[0..12]: [99, 92, 100, 100, 99, 99, 100, 99, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995892492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99590616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995918878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995931647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995944285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995963342Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"":"downstairs","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.99597966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.995991758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"0","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996004936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"1","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996019064Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"client":"2","":"downstairs","session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996031952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"c702a541-f71b-4221-b79c-e7cf4b865fe8 is now active with session: 00ce8a0b-7cdd-4e89-9a6d-b8653df520f6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996056658Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996068657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590,"session_id":"00ce8a0b-7cdd-4e89-9a6d-b8653df520f6"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:26.996080795Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 551 len 4096 data: 7 02/30 Read block 475 len 32768 03/30 Read block 390 len 36864 04/30 Write block 570 len 36864 data: 9 9 9 10 11 11 9 10 8 05/30 Write block 398 len 40960 data: 14 13 12 11 11 15 15 15 14 13 06/30 Flush 07/30 Write block 10 len 12288 data: 6 5 6 08/30 Write block 165 len 12288 data: 9 9 8 09/30 Read block 248 len 24576 10/30 Write block 172 len 36864 data: 6 9 7 6 7 7 7 7 8 11/30 Write block 234 len 40960 data: 11 12 13 10 10 11 12 10 13 9 12/30 Read block 47 len 12288 13/30 Read block 367 len 32768 14/30 Write block 369 len 40960 data: 7 8 7 6 5 6 8 8 10 10 15/30 Read block 587 len 36864 16/30 Read block 76 len 32768 17/30 Write block 424 len 8192 data: 8 10 18/30 Read block 403 len 4096 19/30 Flush 20/30 Flush 21/30 Write block 349 len 32768 data: 12 10 10 10 11 12 12 13 22/30 Write block 111 len 20480 data: 6 6 6 5 4 23/30 Write block 247 len 4096 data: 7 24/30 Write block 354 len 8192 data: 13 13 25/30 Write block 324 len 24576 data: 10 11 10 11 11 14 26/30 Read block 62 len 16384 27/30 Write block 295 len 4096 data: 13 28/30 Write block 168 len 16384 data: 6 8 6 6 29/30 Read block 3 len 16384 30/30 Write block 520 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:27.014568384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 100 100 100 161 161 161 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 99 99 99 159 159 159 T T T 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 102 102 102 168 168 168 T T T 9 180-199 102 102 102 168 168 168 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 102 102 102 168 168 168 F F F 12 240-259 102 102 102 168 168 168 T T T 13 260-279 96 96 96 152 152 152 F F F 14 280-299 96 96 96 154 154 154 T T T 15 300-319 101 101 101 166 166 166 F F F 16 320-339 99 99 99 159 159 159 T T T 17 340-359 100 100 100 162 162 162 T T T 18 360-379 102 102 102 168 168 168 F F F 19 380-399 102 102 102 167 167 167 F F F 20 400-419 102 102 102 167 167 167 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 100 100 100 164 164 164 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 100 100 100 165 165 165 F F F 26 520-539 92 92 92 147 147 147 T T T 27 540-559 102 102 102 167 167 167 F F F 28 560-579 102 102 102 167 167 167 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 102, Max flush: 168 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 100 100 100 161 161 161 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 103 103 103 169 169 169 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 103 103 103 169 169 169 F F F 9 180-199 102 102 102 168 168 168 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 102 102 102 168 168 168 F F F 12 240-259 103 103 103 169 169 169 F F F 13 260-279 96 96 96 152 152 152 F F F 14 280-299 103 103 103 169 169 169 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 103 103 103 169 169 169 F F F 17 340-359 103 103 103 169 169 169 F F F 18 360-379 102 102 102 168 168 168 F F F 19 380-399 102 102 102 167 167 167 F F F 20 400-419 102 102 102 167 167 167 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 100 100 100 164 164 164 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 100 100 100 165 165 165 F F F 26 520-539 103 103 103 169 169 169 F F F 27 540-559 102 102 102 167 167 167 F F F 28 560-579 102 102 102 167 167 167 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 103, Max flush: 169 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.302649947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.346647334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.34729312Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.347337683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.347352521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.347366919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} {"msg":"Crucible c7285e36-f400-4081-a2d6-0d0beaab6768 has session id: de1aca12-0f01-49d4-97e3-2452efdbe543","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.347926898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Upstairs opts: Upstairs UUID: c7285e36-f400-4081-a2d6-0d0beaab6768, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.34798227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Crucible stats registered with UUID: c7285e36-f400-4081-a2d6-0d0beaab6768","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348004136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:31.348020494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.3485298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348572153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} {"msg":"c7285e36-f400-4081-a2d6-0d0beaab6768 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348624456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348644873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348670829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348687587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348701805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348715223Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348739629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348761726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348786173Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348810879Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"ds_connection connected from Ok(127.0.0.1:65398)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348834915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"ds_connection connected from Ok(127.0.0.1:63551)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348858682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"ds_connection connected from Ok(127.0.0.1:44307)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.348883078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.349548202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.349577037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.349739714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.350211925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.350238401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.350491124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.350834614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.350859331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[0]R flush_numbers[0..12]: [168, 147, 161, 161, 159, 169, 162, 159, 169, 168, 158, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351075849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[0]R generation[0..12]: [102, 92, 100, 100, 99, 103, 100, 99, 103, 102, 98, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351104665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351120743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[1]R flush_numbers[0..12]: [168, 147, 161, 161, 159, 169, 162, 159, 169, 168, 158, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.35113508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[1]R generation[0..12]: [102, 92, 100, 100, 99, 103, 100, 99, 103, 102, 98, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351149138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351162916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[2]R flush_numbers[0..12]: [168, 147, 161, 161, 159, 169, 162, 159, 169, 168, 158, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351176414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[2]R generation[0..12]: [102, 92, 100, 100, 99, 103, 100, 99, 103, 102, 98, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351189773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.35120788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351221098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351234056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351257803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351299487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"":"downstairs","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351327023Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351350959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"0","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351377355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"1","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351403461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"client":"2","":"downstairs","session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351429868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351454534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"c7285e36-f400-4081-a2d6-0d0beaab6768 is now active with session: de1aca12-0f01-49d4-97e3-2452efdbe543","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.35148233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351506037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598,"session_id":"de1aca12-0f01-49d4-97e3-2452efdbe543"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:31.351519724Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 20480 data: 12 10 7 10 8 02/30 Read block 331 len 4096 03/30 Flush 04/30 Write block 263 len 12288 data: 6 7 8 05/30 Read block 584 len 4096 06/30 Write block 231 len 20480 data: 6 10 8 12 13 07/30 Write block 537 len 28672 data: 12 12 11 9 9 10 9 08/30 Write block 322 len 36864 data: 6 8 11 12 11 12 12 15 12 09/30 Read block 555 len 8192 10/30 Read block 574 len 28672 11/30 Write block 553 len 4096 data: 8 12/30 Read block 377 len 16384 13/30 Write block 508 len 24576 data: 15 11 9 8 8 7 14/30 Read block 46 len 4096 15/30 Read block 481 len 36864 16/30 Read block 311 len 28672 17/30 Write block 161 len 4096 data: 8 18/30 Read block 167 len 40960 19/30 Write block 380 len 4096 data: 9 20/30 Write block 402 len 16384 data: 12 16 16 16 21/30 Read block 462 len 32768 22/30 Flush 23/30 Write block 281 len 40960 data: 11 10 10 13 15 13 13 12 13 11 24/30 Write block 452 len 8192 data: 7 8 25/30 Read block 145 len 16384 26/30 Read block 588 len 12288 27/30 Read block 525 len 16384 28/30 Write block 45 len 16384 data: 8 9 10 8 29/30 Write block 532 len 40960 data: 9 9 9 12 12 13 13 12 10 10 30/30 Write block 559 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:31.369020248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 100 100 100 161 161 161 T T T 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 103 103 103 169 169 169 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 102 102 102 168 168 168 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 104 104 104 171 171 171 F F F 12 240-259 103 103 103 169 169 169 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 103 103 103 169 169 169 T T T 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 102 102 102 168 168 168 F F F 19 380-399 104 104 104 171 171 171 F F F 20 400-419 104 104 104 171 171 171 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 100 100 100 164 164 164 T T T 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 104 104 104 171 171 171 F F F 26 520-539 104 104 104 171 171 171 T T T 27 540-559 104 104 104 171 171 171 T T T 28 560-579 102 102 102 167 167 167 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 104, Max flush: 171 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 99 99 99 159 159 159 F F F 5 100-119 103 103 103 169 169 169 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 102 102 102 168 168 168 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 104 104 104 171 171 171 F F F 12 240-259 103 103 103 169 169 169 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 102 102 102 168 168 168 F F F 19 380-399 104 104 104 171 171 171 F F F 20 400-419 104 104 104 171 171 171 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 105 105 105 172 172 172 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 F F F 25 500-519 104 104 104 171 171 171 F F F 26 520-539 105 105 105 172 172 172 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 167 167 167 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 105, Max flush: 172 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.605666676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.651283207Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.65202309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.652074402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.65209226Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.652110617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} {"msg":"Crucible 0f7d216c-482a-4609-a732-5acc9ff96f60 has session id: 415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.652720718Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Upstairs opts: Upstairs UUID: 0f7d216c-482a-4609-a732-5acc9ff96f60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.652772591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Crucible stats registered with UUID: 0f7d216c-482a-4609-a732-5acc9ff96f60","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.652796727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:35.652816994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653339098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653363674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} {"msg":"0f7d216c-482a-4609-a732-5acc9ff96f60 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653428575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653445413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653468179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653482247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653496135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653508964Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653522551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653535309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653548678Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653561906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49921)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653653263Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56189)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653703705Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63458)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.653845055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.654506268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.654534424Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.654558021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655126148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655272557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.6553893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655685817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655778123Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[0]R flush_numbers[0..12]: [168, 147, 172, 161, 159, 169, 162, 159, 171, 168, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.65593562Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[0]R generation[0..12]: [102, 92, 105, 100, 99, 103, 100, 99, 104, 102, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655964586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.655981154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[1]R flush_numbers[0..12]: [168, 147, 172, 161, 159, 169, 162, 159, 171, 168, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656022578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[1]R generation[0..12]: [102, 92, 105, 100, 99, 103, 100, 99, 104, 102, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656043235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656069051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[2]R flush_numbers[0..12]: [168, 147, 172, 161, 159, 169, 162, 159, 171, 168, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656095897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[2]R generation[0..12]: [102, 92, 105, 100, 99, 103, 100, 99, 104, 102, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656112925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656135941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid"Wait for a query_work_queue command to finish before sending IO :1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656166357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656192363Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.65621092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656224148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"":"downstairs","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656243266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656264692Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"0","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656289699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"1","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656308166Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"client":"2","":"downstairs","session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656322224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.65634608Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"0f7d216c-482a-4609-a732-5acc9ff96f60 is now active with session: 415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656370367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656392994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606,"session_id":"415f2610-ddb8-4a9b-a2b5-3e00c6ea0bd3"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:35.656411031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 84 len 16384 data: 3 3 4 5 02/30 Read block 590 len 28672 03/30 Write block 504 len 32768 data: 13 13 13 16 16 12 10 9 04/30 Read block 259 len 36864 05/30 Write block 344 len 16384 data: 9 11 11 10 06/30 Write block 411 len 16384 data: 9 11 9 10 07/30 Write block 383 len 40960 data: 8 5 7 6 7 9 11 12 10 12 08/30 Read block 133 len 32768 09/30 Write block 518 len 16384 data: 9 11 10 9 10/30 Write block 401 len 4096 data: 12 11/30 Read block 75 len 8192 12/30 Read block 271 len 12288 13/30 Write block 545 len 4096 data: 7 14/30 Read block 150 len 36864 15/30 Read block 268 len 32768 16/30 Read block 512 len 24576 17/30 Read block 165 len 28672 18/30 Flush 19/30 Write block 499 len 36864 data: 11 11 12 11 13 14 14 14 17 20/30 Read block 192 len 28672 21/30 Read block 275 len 12288 22/30 Read block 90 len 12288 23/30 Write block 189 len 32768 data: 13 12 12 10 9 9 9 10 24/30 Write block 438 len 8192 data: 8 7 25/30 Write block 556 len 20480 data: 10 8 9 9 6 26/30 Write block 562 len 8192 data: 6 6 27/30 Read block 160 len 36864 28/30 Write block 350 len 16384 data: 11 11 11 12 29/30 Read block 378 len 28672 30/30 Read block 196 len 8192 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:35.675285294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 106 106 106 173 173 173 F F F 5 100-119 103 103 103 169 169 169 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 102 102 102 168 168 168 T T T 10 200-219 98 98 98 158 158 158 F F F 11 220-239 104 104 104 171 171 171 F F F 12 240-259 103 103 103 169 169 169 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 106 106 106 173 173 173 T T T 18 360-379 102 102 102 168 168 168 F F F 19 380-399 106 106 106 173 173 173 F F F 20 400-419 106 106 106 173 173 173 F F F 21 420-439 102 102 102 168 168 168 T T T 22 440-459 105 105 105 172 172 172 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 100 100 100 165 165 165 T T T 25 500-519 106 106 106 173 173 173 T T T 26 520-539 106 106 106 173 173 173 F F F 27 540-559 106 106 106 173 173 173 T T T 28 560-579 102 102 102 167 167 167 T T T 29 580-599 95 95 95 151 151 151 F F F Max gen: 106, Max flush: 173 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 168 168 168 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 106 106 106 173 173 173 F F F 5 100-119 103 103 103 169 169 169 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 99 99 99 159 159 159 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 104 104 104 171 171 171 F F F 12 240-259 103 103 103 169 169 169 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 102 102 102 168 168 168 F F F 19 380-399 106 106 106 173 173 173 F F F 20 400-419 106 106 106 173 173 173 F F F 21 420-439 107 107 107 174 174 174 F F F 22 440-459 105 105 105 172 172 172 F F F 23 460-479 99 99 99 159 159 159 F F F 24 480-499 107 107 107 174 174 174 F F F 25 500-519 107 107 107 174 174 174 F F F 26 520-539 106 106 106 173 173 173 F F F 27 540-559 107 107 107 174 174 174 F F F 28 560-579 107 107 107 174 174 174 F F F 29 580-599 95 95 95 151 151 151 F F F Max gen: 107, Max flush: 174 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.924871629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.968417611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969011835Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969055288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969071246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969085854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} {"msg":"Crucible cf2c5bc5-a84d-48ec-a5aa-17627eddd3d6 has session id: 7ea33c23-f0d2-46aa-ba86-ff8ce721d881","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969634844Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Upstairs opts: Upstairs UUID: cf2c5bc5-a84d-48ec-a5aa-17627eddd3d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969667909Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Crucible stats registered with UUID: cf2c5bc5-a84d-48ec-a5aa-17627eddd3d6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.969682897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:39.969695775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970138091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970171736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} {"msg":"cf2c5bc5-a84d-48ec-a5aa-17627eddd3d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970225118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970255234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970292958Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970320204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.97035202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970377956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970406392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970431258Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970457624Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.97048456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"ds_connection connected from Ok(127.0.0.1:34815)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970512036Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"ds_connection connected from Ok(127.0.0.1:50711)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970540352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"ds_connection connected from Ok(127.0.0.1:42411)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.970569098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.971199346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.971384339Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.971912863Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.971947857Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.972031175Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.972390673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.972457193Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.972637307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[0]R flush_numbers[0..12]: [168, 147, 172, 161, 173, 169, 162, 159, 171, 174, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973120917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[0]R generation[0..12]: [102, 92, 105, 100, 106, 103, 100, 99, 104, 107, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973140094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973154722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[1]R flush_numbers[0..12]: [168, 147, 172, 161, 173, 169, 162, 159, 171, 174, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.97316827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[1]R generation[0..12]: [102, 92, 105, 100, 106, 103, 100, 99, 104, 107, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973181798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973195506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[2]R flush_numbers[0..12]: [168, 147, 172, 161, 173, 169, 162, 159, 171, 174, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973208854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[2]R generation[0..12]: [102, 92, 105, 100, 106, 103, 100, 99, 104, 107, 98, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973222132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.9732357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973248208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973261186Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973277244Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973296501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"":"downstairs","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973309509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973321087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"0","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973333906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"1","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973346404Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"client":"2","":"downstairs","session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973359012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.97337081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"cf2c5bc5-a84d-48ec-a5aa-17627eddd3d6 is now active with session: 7ea33c23-f0d2-46aa-ba86-ff8ce721d881","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973383438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973395097Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614,"session_id":"7ea33c23-f0d2-46aa-ba86-ff8ce721d881"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:39.973406875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 32768 data: 16 11 11 8 8 7 7 9 02/30 Flush 03/30 Write block 9 len 4096 data: 5 04/30 Write block 419 len 16384 data: 11 11 10 11 05/30 Write block 102 len 32768 data: 8 8 11 9 7 10 9 7 06/30 Write block 582 len 4096 data: 9 07/30 Write block 415 len 32768 data: 10 10 9 10 12 12 11 12 08/30 Write block 229 len 20480 data: 5 5 7 11 9 09/30 Read block 583 len 4096 10/30 Read block 581 len 20480 11/30 Read block 12 len 16384 12/30 Write block 111 len 28672 data: 7 7 7 6 5 4 4 13/30 Write block 361 len 36864 data: 9 10 9 8 8 9 9 8 8 14/30 Read block 472 len 16384 15/30 Flush 16/30 Write block 148 len 20480 data: 13 17 12 12 9 17/30 Write block 386 len 24576 data: 7 8 10 12 13 11 18/30 Write block 460 len 40960 data: 8 6 8 8 8 8 9 8 10 8 19/30 Write block 257 len 4096 data: 9 20/30 Read block 386 len 8192 21/30 Write block 430 len 12288 data: 11 10 8 22/30 Write block 384 len 12288 data: 6 8 8 23/30 Read block 89 len 24576 24/30 Write block 141 len 20480 data: 6 8 10 11 13 25/30 Read block 98 len 20480 26/30 Write block 145 len 20480 data: 14 11 12 14 18 27/30 Write block 7 len 16384 data: 6 6 6 7 28/30 Write block 238 len 12288 data: 11 12 13 29/30 Write block 97 len 32768 data: 9 10 12 12 12 9 9 12 30/30 Read block 384 len 4096 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:39.990112974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 176 176 176 T T T 1 020-039 92 92 92 147 147 147 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 106 106 106 173 173 173 T T T 5 100-119 108 108 108 176 176 176 T T T 6 120-139 100 100 100 162 162 162 F F F 7 140-159 108 108 108 175 175 175 T T T 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 108 108 108 176 176 176 T T T 12 240-259 103 103 103 169 169 169 T T T 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 108 108 108 176 176 176 F F F 19 380-399 106 106 106 173 173 173 T T T 20 400-419 108 108 108 176 176 176 F F F 21 420-439 108 108 108 176 176 176 T T T 22 440-459 105 105 105 172 172 172 F F F 23 460-479 99 99 99 159 159 159 T T T 24 480-499 107 107 107 174 174 174 F F F 25 500-519 107 107 107 174 174 174 F F F 26 520-539 106 106 106 173 173 173 F F F 27 540-559 107 107 107 174 174 174 F F F 28 560-579 107 107 107 174 174 174 F F F 29 580-599 108 108 108 176 176 176 F F F Max gen: 108, Max flush: 176 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 177 177 177 F F F 1 020-039 92 92 92 147 147 147 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 109 109 109 177 177 177 F F F 5 100-119 109 109 109 177 177 177 F F F 6 120-139 100 100 100 162 162 162 F F F 7 140-159 109 109 109 177 177 177 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 109 109 109 177 177 177 F F F 12 240-259 109 109 109 177 177 177 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 108 108 108 176 176 176 F F F 19 380-399 109 109 109 177 177 177 F F F 20 400-419 108 108 108 176 176 176 F F F 21 420-439 109 109 109 177 177 177 F F F 22 440-459 105 105 105 172 172 172 F F F 23 460-479 109 109 109 177 177 177 F F F 24 480-499 107 107 107 174 174 174 F F F 25 500-519 107 107 107 174 174 174 F F F 26 520-539 106 106 106 173 173 173 F F F 27 540-559 107 107 107 174 174 174 F F F 28 560-579 107 107 107 174 174 174 F F F 29 580-599 108 108 108 176 176 176 F F F Max gen: 109, Max flush: 177 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.243939062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.288382433Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.288966348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.289007053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.28902261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.289037558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} {"msg":"Crucible 3e1e613b-f08c-455a-8b27-544a1bc895c4 has session id: 8e0f8bd6-a836-4070-864a-f2e8f40d9795","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.289615484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Upstairs opts: Upstairs UUID: 3e1e613b-f08c-455a-8b27-544a1bc895c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.289660087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Crucible stats registered with UUID: 3e1e613b-f08c-455a-8b27-544a1bc895c4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.289690063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:44.28971209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290209028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290234104Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} {"msg":"3e1e613b-f08c-455a-8b27-544a1bc895c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290294205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290312203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290336079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290349907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290363905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290380743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290442924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290492177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290528301Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290545579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"ds_connection connected from Ok(127.0.0.1:43921)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290560107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"ds_connection connected from Ok(127.0.0.1:52710)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290828577Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"ds_connection connected from Ok(127.0.0.1:33236)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.290864602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.291314017Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.291352241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.291471374Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.291932377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.291971241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292242842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292504624Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292615288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[0]R flush_numbers[0..12]: [177, 147, 172, 161, 177, 177, 162, 177, 171, 174, 158, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.29273753Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[0]R generation[0..12]: [109, 92, 105, 100, 109, 109, 100, 109, 104, 107, 98, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292759327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292774454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[1]R flush_numbers[0..12]: [177, 147, 172, 161, 177, 177, 162, 177, 171, 174, 158, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292788232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[1]R generation[0..12]: [109, 92, 105, 100, 109, 109, 100, 109, 104, 107, 98, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.29280535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292819568Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[2]R flush_numbers[0..12]: [177, 147, 172, 161, 177, 177, 162, 177, 171, 174, 158, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292833256Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[2]R generation[0..12]: [109, 92, 105, 100, 109, 109, 100, 109, 104, 107, 98, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292846864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292860602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.29287329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:14:44.292886388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292903446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292916314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"":"downstairs","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292929102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.2929409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"0","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292954048Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"1","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292966806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"client":"2","":"downstairs","session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292979814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.292991943Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"3e1e613b-f08c-455a-8b27-544a1bc895c4 is now active with session: 8e0f8bd6-a836-4070-864a-f2e8f40d9795","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.29300763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.293019838Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622,"session_id":"8e0f8bd6-a836-4070-864a-f2e8f40d9795"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:44.293031627Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 28672 data: 12 13 14 11 14 10 7 02/30 Read block 308 len 8192 03/30 Read block 69 len 4096 04/30 Read block 41 len 28672 05/30 Read block 524 len 8192 06/30 Write block 390 len 20480 data: 14 12 13 11 8 07/30 Read block 137 len 32768 08/30 Read block 220 len 12288 09/30 Write block 503 len 28672 data: 14 15 15 15 18 17 13 10/30 Write block 137 len 32768 data: 6 5 5 4 7 9 11 12 11/30 Read block 180 len 20480 12/30 Read block 511 len 32768 13/30 Write block 100 len 36864 data: 13 13 10 10 13 10 8 11 10 14/30 Read block 213 len 20480 15/30 Write block 22 len 40960 data: 3 4 4 4 4 4 7 5 7 9 16/30 Write block 3 len 20480 data: 7 7 6 5 7 17/30 Write block 508 len 12288 data: 18 14 11 18/30 Read block 263 len 32768 19/30 Read block 563 len 28672 20/30 Read block 486 len 4096 21/30 Read block 265 len 36864 22/30 Flush 23/30 Flush 24/30 Write block 235 len 28672 data: 14 14 11 13 14 15 12 25/30 Write block 474 len 32768 data: 9 11 10 10 10 9 8 8 26/30 Write block 462 len 36864 data: 9 9 9 9 10 9 11 9 9 27/30 Write block 555 len 40960 data: 10 11 9 10 10 7 6 7 7 5 28/30 Write block 150 len 4096 data: 13 29/30 Read block 166 len 40960 30/30 Write block 387 len 8192 data: 9 11 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 6 6 4 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:44.312678008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 110 110 110 178 178 178 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 109 109 109 177 177 177 F F F 5 100-119 110 110 110 178 178 178 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 110 110 110 178 178 178 T T T 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 110 110 110 178 178 178 T T T 12 240-259 110 110 110 178 178 178 T T T 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 108 108 108 176 176 176 F F F 19 380-399 110 110 110 178 178 178 T T T 20 400-419 108 108 108 176 176 176 F F F 21 420-439 109 109 109 177 177 177 F F F 22 440-459 105 105 105 172 172 172 F F F 23 460-479 109 109 109 177 177 177 T T T 24 480-499 107 107 107 174 174 174 T T T 25 500-519 110 110 110 178 178 178 F F F 26 520-539 106 106 106 173 173 173 F F F 27 540-559 107 107 107 174 174 174 T T T 28 560-579 107 107 107 174 174 174 T T T 29 580-599 108 108 108 176 176 176 F F F Max gen: 110, Max flush: 178 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 110 110 110 178 178 178 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 109 109 109 177 177 177 F F F 5 100-119 110 110 110 178 178 178 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 111 111 111 179 179 179 F F F 12 240-259 111 111 111 179 179 179 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 108 108 108 176 176 176 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 108 108 108 176 176 176 F F F 21 420-439 109 109 109 177 177 177 F F F 22 440-459 105 105 105 172 172 172 F F F 23 460-479 111 111 111 179 179 179 F F F 24 480-499 111 111 111 179 179 179 F F F 25 500-519 110 110 110 178 178 178 F F F 26 520-539 106 106 106 173 173 173 F F F 27 540-559 111 111 111 179 179 179 F F F 28 560-579 111 111 111 179 179 179 F F F 29 580-599 108 108 108 176 176 176 F F F Max gen: 111, Max flush: 179 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.567346033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.611355858Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.611986076Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612038619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612060576Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612076823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} {"msg":"Crucible 64c04487-9423-4637-a339-05fcccb4af95 has session id: 7de89f48-827f-4a47-a7c8-4eb666d623d0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612620934Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Upstairs opts: Upstairs UUID: 64c04487-9423-4637-a339-05fcccb4af95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612656559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Crucible stats registered with UUID: 64c04487-9423-4637-a339-05fcccb4af95","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.612673656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:48.612687174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613118071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613144298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} {"msg":"64c04487-9423-4637-a339-05fcccb4af95 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613209328Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613227006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613250512Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613271929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613297875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613322962Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613351118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613377434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.61340574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613432736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43129)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613460142Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54390)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613484568Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42964)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.613507265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.614116056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.614144522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.614744785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.614857748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.615232334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.615448922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.616059563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.616542813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[0]R flush_numbers[0..12]: [178, 178, 172, 161, 177, 178, 178, 179, 171, 174, 158, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617024523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[0]R generation[0..12]: [110, 110, 105, 100, 109, 110, 110, 111, 104, 107, 98, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617048459Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617063137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[1]R flush_numbers[0..12]: [178, 178, 172, 161, 177, 178, 178, 179, 171, 174, 158, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617076815Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[1]R generation[0..12]: [110, 110, 105, 100, 109, 110, 110, 111, 104, 107, 98, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617090323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617104211Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[2]R flush_numbers[0..12]: [178, 178, 172, 161, 177, 178, 178, 179, 171, 174, 158, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617117919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[2]R generation[0..12]: [110, 110, 105, 100, 109, 110, 110, 111, 104, 107, 98, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617131817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617146405Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617159293Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617172072Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.6171847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":Wait for a query_work_queue command to finish before sending IO 1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617203787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"":"downstairs","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617219634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617231513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"0","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617244391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"1","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617257269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"client":"2","":"downstairs","session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617270297Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617282505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"64c04487-9423-4637-a339-05fcccb4af95 is now active with session: 7de89f48-827f-4a47-a7c8-4eb666d623d0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617294884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.617306462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630,"session_id":"7de89f48-827f-4a47-a7c8-4eb666d623d0"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:48.61731823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 8192 data: 12 9 02/30 Read block 502 len 8192 03/30 Read block 428 len 12288 04/30 Write block 530 len 8192 data: 11 8 05/30 Read block 7 len 8192 06/30 Read block 375 len 36864 07/30 Write block 222 len 4096 data: 6 08/30 Read block 336 len 36864 09/30 Read block 549 len 28672 10/30 Read block 563 len 40960 11/30 Read block 414 len 20480 12/30 Flush 13/30 Write block 326 len 20480 data: 12 13 13 16 13 14/30 Read block 271 len 28672 15/30 Write block 339 len 4096 data: 13 16/30 Read block 369 len 8192 17/30 Write block 453 len 4096 data: 9 18/30 Write block 107 len 12288 data: 12 11 8 19/30 Write block 29 len 40960 data: 6 8 10 13 11 11 9 9 10 12 20/30 Read block 304 len 8192 21/30 Read block 377 len 16384 22/30 Read block 216 len 40960 23/30 Read block 450 len 24576 24/30 Read block 580 len 36864 25/30 Write block 591 len 16384 data: 6 5 5 6 26/30 Write block 428 len 40960 data: 10 10 12 11 9 8 8 12 10 9 27/30 Write block 96 len 12288 data: 11 10 11 28/30 Write block 361 len 4096 data: 10 29/30 Read block 446 len 12288 30/30 Read block 500 len 36864 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:48.635067767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 110 110 110 178 178 178 T T T 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 109 109 109 177 177 177 T T T 5 100-119 110 110 110 178 178 178 T T T 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 112 112 112 180 180 180 F F F 12 240-259 111 111 111 179 179 179 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 171 171 171 T T T 17 340-359 107 107 107 174 174 174 F F F 18 360-379 108 108 108 176 176 176 T T T 19 380-399 112 112 112 180 180 180 F F F 20 400-419 108 108 108 176 176 176 F F F 21 420-439 109 109 109 177 177 177 T T T 22 440-459 105 105 105 172 172 172 T T T 23 460-479 111 111 111 179 179 179 F F F 24 480-499 111 111 111 179 179 179 F F F 25 500-519 110 110 110 178 178 178 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 111 111 111 179 179 179 F F F 28 560-579 111 111 111 179 179 179 F F F 29 580-599 108 108 108 176 176 176 T T T Max gen: 112, Max flush: 180 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 105 105 105 172 172 172 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 F F F 5 100-119 113 113 113 181 181 181 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 112 112 112 180 180 180 F F F 12 240-259 111 111 111 179 179 179 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 F F F 17 340-359 107 107 107 174 174 174 F F F 18 360-379 113 113 113 181 181 181 F F F 19 380-399 112 112 112 180 180 180 F F F 20 400-419 108 108 108 176 176 176 F F F 21 420-439 113 113 113 181 181 181 F F F 22 440-459 113 113 113 181 181 181 F F F 23 460-479 111 111 111 179 179 179 F F F 24 480-499 111 111 111 179 179 179 F F F 25 500-519 110 110 110 178 178 178 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 111 111 111 179 179 179 F F F 28 560-579 111 111 111 179 179 179 F F F 29 580-599 113 113 113 181 181 181 F F F Max gen: 113, Max flush: 181 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.890756443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.935479704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936091425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936132989Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936147767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936161825Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} {"msg":"Crucible 9818fa14-849d-4369-a2f9-da42eb1ca9b8 has session id: 526f7a01-23ca-4740-b968-064bfeedf9f9","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936870382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Upstairs opts: Upstairs UUID: 9818fa14-849d-4369-a2f9-da42eb1ca9b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936915365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Crucible stats registered with UUID: 9818fa14-849d-4369-a2f9-da42eb1ca9b8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.936934442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:52.936947351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937400525Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} {"msg":"9818fa14-849d-4369-a2f9-da42eb1ca9b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937526346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937552473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937576799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937591127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937605245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937618593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937633001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937645999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937754973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937790888Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37255)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937818634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34741)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.937841401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59278)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.93791195Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.938438094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.938478918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.939066392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.939224549Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.939263663Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.93969787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.939731895Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.939882764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[0]R flush_numbers[0..12]: [178, 181, 172, 161, 181, 181, 178, 179, 171, 174, 158, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940401288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[0]R generation[0..12]: [110, 113, 105, 100, 113, 113, 110, 111, 104, 107, 98, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940422005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940436503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[1]R flush_numbers[0..12]: [178, 181, 172, 161, 181, 181, 178, 179, 171, 174, 158, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940450181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[1]R generation[0..12]: [110, 113, 105, 100, 113, 113, 110, 111, 104, 107, 98, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940463759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940477457Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[2]R flush_numbers[0..12]: [178, 181, 172, 161, 181, 181, 178, 179, 171, 174, 158, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940490765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[2]R generation[0..12]: [110, 113, 105, 100, 113, 113, 110, 111, 104, 107, 98, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940508982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.94052325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940535948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940548787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B"Wait for a query_work_queue command to finish before sending IO ,"pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940568054Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940581002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"":"downstairs","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.94059398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940605708Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"0","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940618576Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"1","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940631204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"client":"2","":"downstairs","session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940643983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940655841Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"9818fa14-849d-4369-a2f9-da42eb1ca9b8 is now active with session: 526f7a01-23ca-4740-b968-064bfeedf9f9","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940668179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940682907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638,"session_id":"526f7a01-23ca-4740-b968-064bfeedf9f9"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:52.940695155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 576 len 24576 data: 10 11 9 7 6 9 02/30 Read block 83 len 36864 03/30 Read block 82 len 20480 04/30 Flush 05/30 Write block 40 len 16384 data: 14 10 8 8 06/30 Read block 80 len 20480 07/30 Flush 08/30 Write block 223 len 32768 data: 6 4 4 7 6 5 6 6 09/30 Flush 10/30 Write block 571 len 28672 data: 10 10 11 12 12 11 12 11/30 Write block 412 len 16384 data: 12 10 11 11 12/30 Read block 181 len 24576 13/30 Read block 212 len 4096 14/30 Read block 461 len 20480 15/30 Read block 382 len 28672 16/30 Read block 573 len 20480 17/30 Write block 358 len 16384 data: 10 8 8 11 18/30 Read block 411 len 8192 19/30 Read block 494 len 12288 20/30 Read block 160 len 12288 21/30 Read block 67 len 28672 22/30 Write block 388 len 40960 data: 12 13 15 13 14 13 10 11 11 13 23/30 Read block 311 len 24576 24/30 Read block 362 len 28672 25/30 Read block 452 len 20480 26/30 Read block 450 len 4096 27/30 Read block 245 len 8192 28/30 Write block 451 len 24576 data: 8 8 10 10 10 8 29/30 Write block 381 len 8192 data: 9 9 30/30 Read block 48 len 32768 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 20 20 21 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:52.960542877Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 F F F 5 100-119 113 113 113 181 181 181 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 114 114 114 184 184 184 F F F 12 240-259 111 111 111 179 179 179 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 F F F 17 340-359 107 107 107 174 174 174 T T T 18 360-379 113 113 113 181 181 181 T T T 19 380-399 112 112 112 180 180 180 T T T 20 400-419 108 108 108 176 176 176 T T T 21 420-439 113 113 113 181 181 181 F F F 22 440-459 113 113 113 181 181 181 T T T 23 460-479 111 111 111 179 179 179 F F F 24 480-499 111 111 111 179 179 179 F F F 25 500-519 110 110 110 178 178 178 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 111 111 111 179 179 179 F F F 28 560-579 114 114 114 182 182 182 T T T 29 580-599 114 114 114 182 182 182 F F F Max gen: 114, Max flush: 184 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 F F F 5 100-119 113 113 113 181 181 181 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 114 114 114 184 184 184 F F F 12 240-259 111 111 111 179 179 179 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 F F F 17 340-359 115 115 115 185 185 185 F F F 18 360-379 115 115 115 185 185 185 F F F 19 380-399 115 115 115 185 185 185 F F F 20 400-419 115 115 115 185 185 185 F F F 21 420-439 113 113 113 181 181 181 F F F 22 440-459 115 115 115 185 185 185 F F F 23 460-479 111 111 111 179 179 179 F F F 24 480-499 111 111 111 179 179 179 F F F 25 500-519 110 110 110 178 178 178 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 111 111 111 179 179 179 F F F 28 560-579 115 115 115 185 185 185 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 115, Max flush: 185 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.196861372Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.240310819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.24092469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.240975612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.240998639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.241017146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} {"msg":"Crucible 15bcea1a-25b1-41d7-923f-1b28342fcbc9 has session id: 3b83df1c-62ea-4b5d-afbd-63e14c59fb23","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.241636136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Upstairs opts: Upstairs UUID: 15bcea1a-25b1-41d7-923f-1b28342fcbc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.24168094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Crucible stats registered with UUID: 15bcea1a-25b1-41d7-923f-1b28342fcbc9","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.241705106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:57.241726353Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242116246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242163729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} {"msg":"15bcea1a-25b1-41d7-923f-1b28342fcbc9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242190165Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242211202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242243418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242265224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242289791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242315777Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242341233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.24236852Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242397175Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242426501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"ds_connection connected from Ok(127.0.0.1:64580)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242473414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"ds_connection connected from Ok(127.0.0.1:60873)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.24249937Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"ds_connection connected from Ok(127.0.0.1:58173)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.242737566Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.243254771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.243418667Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.243453182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.243938621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.243967477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244021089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244529115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244553781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[0]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 181, 178, 179, 171, 174, 158, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244582607Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[0]R generation[0..12]: [110, 113, 114, 100, 113, 113, 110, 111, 104, 107, 98, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244598895Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244613163Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[1]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 181, 178, 179, 171, 174, 158, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244626701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[1]R generation[0..12]: [110, 113, 114, 100, 113, 113, 110, 111, 104, 107, 98, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244639889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244653377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[2]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 181, 178, 179, 171, 174, 158, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244666695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[2]R generation[0..12]: [110, 113, 114, 100, 113, 113, 110, 111, 104, 107, 98, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244681133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244694721Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244707419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244720117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244732655Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244745203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"":"downstairs","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244758401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:14:57.24477223Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"0","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244788867Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"1","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244801865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"client":"2","":"downstairs","session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244814713Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244826552Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"15bcea1a-25b1-41d7-923f-1b28342fcbc9 is now active with session: 3b83df1c-62ea-4b5d-afbd-63e14c59fb23","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.24483908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244850748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646,"session_id":"3b83df1c-62ea-4b5d-afbd-63e14c59fb23"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:14:57.244862617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 230 len 32768 data: 7 8 12 10 13 15 15 12 03/30 Flush 04/30 Write block 499 len 24576 data: 12 12 13 12 15 16 05/30 Read block 192 len 12288 06/30 Read block 536 len 4096 07/30 Read block 270 len 28672 08/30 Read block 395 len 8192 09/30 Read block 58 len 36864 10/30 Write block 490 len 20480 data: 6 6 5 3 3 11/30 Read block 499 len 12288 12/30 Read block 382 len 24576 13/30 Write block 243 len 16384 data: 11 8 8 7 14/30 Read block 353 len 40960 15/30 Read block 282 len 4096 16/30 Read block 392 len 28672 17/30 Write block 390 len 40960 data: 16 14 15 14 11 12 12 14 15 14 18/30 Write block 347 len 16384 data: 11 10 13 12 19/30 Read block 149 len 24576 20/30 Read block 237 len 20480 21/30 Flush 22/30 Write block 109 len 24576 data: 9 7 8 8 8 7 23/30 Read block 96 len 32768 24/30 Read block 295 len 12288 25/30 Read block 464 len 12288 26/30 Write block 193 len 4096 data: 10 27/30 Write block 543 len 20480 data: 10 8 8 7 9 28/30 Write block 582 len 20480 data: 10 6 10 11 11 29/30 Read block 568 len 32768 30/30 Write block 457 len 36864 data: 9 7 8 9 7 10 10 10 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:14:57.26442126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 F F F 5 100-119 113 113 113 181 181 181 T T T 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 107 107 107 174 174 174 T T T 10 200-219 98 98 98 158 158 158 F F F 11 220-239 116 116 116 187 187 187 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 F F F 17 340-359 116 116 116 188 188 188 F F F 18 360-379 115 115 115 185 185 185 F F F 19 380-399 116 116 116 188 188 188 F F F 20 400-419 115 115 115 185 185 185 F F F 21 420-439 113 113 113 181 181 181 F F F 22 440-459 115 115 115 185 185 185 T T T 23 460-479 111 111 111 179 179 179 T T T 24 480-499 116 116 116 188 188 188 F F F 25 500-519 116 116 116 188 188 188 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 111 111 111 179 179 179 T T T 28 560-579 115 115 115 185 185 185 F F F 29 580-599 114 114 114 182 182 182 T T T Max gen: 116, Max flush: 188 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 98 98 98 158 158 158 F F F 11 220-239 116 116 116 187 187 187 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 F F F 17 340-359 116 116 116 188 188 188 F F F 18 360-379 115 115 115 185 185 185 F F F 19 380-399 116 116 116 188 188 188 F F F 20 400-419 115 115 115 185 185 185 F F F 21 420-439 113 113 113 181 181 181 F F F 22 440-459 117 117 117 189 189 189 F F F 23 460-479 117 117 117 189 189 189 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 116 116 116 188 188 188 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 117 117 117 189 189 189 F F F 28 560-579 115 115 115 185 185 185 F F F 29 580-599 117 117 117 189 189 189 F F F Max gen: 117, Max flush: 189 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.502187385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.548926133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.549560951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.549612553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.549630021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.549645048Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} {"msg":"Crucible 4778f6be-1796-4173-8855-87f5715bf9d5 has session id: 1941536c-089e-42b7-aafa-4571414e7a76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550159893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Upstairs opts: Upstairs UUID: 4778f6be-1796-4173-8855-87f5715bf9d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550192579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Crucible stats registered with UUID: 4778f6be-1796-4173-8855-87f5715bf9d5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550207706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:01.550220535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550626386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550651322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} {"msg":"4778f6be-1796-4173-8855-87f5715bf9d5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550713013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550743779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550776424Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.55080003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550827826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550853412Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550881868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550907805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550933871Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550957177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"ds_connection connected from Ok(127.0.0.1:52330)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.550982144Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"ds_connection connected from Ok(127.0.0.1:58802)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.55100682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"ds_connection connected from Ok(127.0.0.1:61475)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.551051794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.55162615Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.55176267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.55224778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.552279245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.552295913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.552787891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.552904914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.552943558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[0]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 189, 178, 179, 171, 189, 158, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553459423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[0]R generation[0..12]: [110, 113, 114, 100, 113, 117, 110, 111, 104, 117, 98, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553487459Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553502667Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[1]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 189, 178, 179, 171, 189, 158, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553516515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[1]R generation[0..12]: [110, 113, 114, 100, 113, 117, 110, 111, 104, 117, 98, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553530213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553544181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[2]R flush_numbers[0..12]: [178, 181, 183, 161, 181, 189, 178, 179, 171, 189, 158, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553558059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[2]R generation[0..12]: [110, 113, 114, 100, 113, 117, 110, 111, 104, 117, 98, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553571567Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553585235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553602723Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553616361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553631278Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"All extents match","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-01-14T16:15:01.553652945Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"":"downstairs","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553669453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.5536841Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"0","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553700388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"1","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553716416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"client":"2","":"downstairs","session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553732913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553747931Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"4778f6be-1796-4173-8855-87f5715bf9d5 is now active with session: 1941536c-089e-42b7-aafa-4571414e7a76","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553763609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553777977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654,"session_id":"1941536c-089e-42b7-aafa-4571414e7a76"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:01.553792855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 40960 data: 11 9 10 12 12 12 11 14 13 12 02/30 Read block 307 len 36864 03/30 Write block 224 len 36864 data: 5 5 8 7 6 7 8 9 13 04/30 Read block 287 len 36864 05/30 Write block 537 len 32768 data: 14 14 13 11 11 11 11 9 06/30 Read block 526 len 16384 07/30 Write block 96 len 16384 data: 12 11 12 13 08/30 Write block 144 len 20480 data: 13 15 12 13 15 09/30 Read block 116 len 8192 10/30 Write block 24 len 36864 data: 5 5 5 5 8 7 9 11 14 11/30 Write block 458 len 20480 data: 8 9 10 8 11 12/30 Write block 515 len 24576 data: 7 7 8 10 12 11 13/30 Read block 336 len 28672 14/30 Write block 371 len 4096 data: 8 15/30 Write block 219 len 36864 data: 5 6 7 7 7 6 6 9 8 16/30 Read block 177 len 40960 17/30 Read block 389 len 12288 18/30 Write block 91 len 8192 data: 5 6 19/30 Read block 216 len 12288 20/30 Read block 180 len 32768 21/30 Write block 545 len 28672 data: 9 8 10 11 10 7 8 22/30 Read block 106 len 12288 23/30 Write block 469 len 40960 data: 10 10 8 11 10 10 12 11 11 11 24/30 Read block 283 len 8192 25/30 Read block 493 len 24576 26/30 Read block 267 len 32768 27/30 Read block 240 len 24576 28/30 Write block 338 len 28672 data: 13 14 11 8 12 10 11 29/30 Read block 378 len 16384 30/30 Write block 423 len 8192 data: 9 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 29 29 27 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:01.573893319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 T T T 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 113 113 113 181 181 181 T T T 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 111 111 111 179 179 179 T T T 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 98 98 98 158 158 158 T T T 11 220-239 116 116 116 187 187 187 T T T 12 240-259 116 116 116 188 188 188 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 113 113 113 181 181 181 T T T 17 340-359 116 116 116 188 188 188 T T T 18 360-379 115 115 115 185 185 185 T T T 19 380-399 116 116 116 188 188 188 F F F 20 400-419 115 115 115 185 185 185 F F F 21 420-439 113 113 113 181 181 181 T T T 22 440-459 117 117 117 189 189 189 T T T 23 460-479 117 117 117 189 189 189 T T T 24 480-499 116 116 116 188 188 188 F F F 25 500-519 116 116 116 188 188 188 T T T 26 520-539 112 112 112 180 180 180 T T T 27 540-559 117 117 117 189 189 189 T T T 28 560-579 115 115 115 185 185 185 F F F 29 580-599 117 117 117 189 189 189 F F F Max gen: 117, Max flush: 189 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 118 118 118 190 190 190 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 118 118 118 190 190 190 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 118 118 118 190 190 190 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 104 104 104 171 171 171 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 118 118 118 190 190 190 F F F 17 340-359 118 118 118 190 190 190 F F F 18 360-379 118 118 118 190 190 190 F F F 19 380-399 116 116 116 188 188 188 F F F 20 400-419 115 115 115 185 185 185 F F F 21 420-439 118 118 118 190 190 190 F F F 22 440-459 118 118 118 190 190 190 F F F 23 460-479 118 118 118 190 190 190 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 118 118 118 190 190 190 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 118 118 118 190 190 190 F F F 28 560-579 115 115 115 185 185 185 F F F 29 580-599 117 117 117 189 189 189 F F F Max gen: 118, Max flush: 190 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.861552802Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.905276458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.905847535Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.905889919Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.905909126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.905924014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} {"msg":"Crucible 95717424-86dc-4fb2-bab2-ab1a197aff04 has session id: 6e5754f9-c6b1-4e46-9a11-390887185c4b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.906446458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Upstairs opts: Upstairs UUID: 95717424-86dc-4fb2-bab2-ab1a197aff04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.906489252Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Crucible stats registered with UUID: 95717424-86dc-4fb2-bab2-ab1a197aff04","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.906513038Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:15.906533325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.906903362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.906928098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} {"msg":"95717424-86dc-4fb2-bab2-ab1a197aff04 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907000447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907018005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907040411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907054249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907068267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907080916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907098793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907111951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907125269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907138407Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37683)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907222735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:15.907265109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:15.907296024Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55251)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907322181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:15.907351866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907435594Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.90787301Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.907910775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.908529665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.908558581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.909166292Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:15.909201587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.909341086Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56166)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.909603178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.910544511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911137394Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[0]R flush_numbers[0..12]: [178, 190, 183, 161, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911790319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[0]R generation[0..12]: [110, 118, 114, 100, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911810676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911825254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[1]R flush_numbers[0..12]: [178, 190, 183, 161, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911839202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[1]R generation[0..12]: [110, 118, 114, 100, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911852741Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911866528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[2]R flush_numbers[0..12]: [178, 190, 183, 161, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911879757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[2]R generation[0..12]: [110, 118, 114, 100, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911893075Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911911102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.91192434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Generation requested: 120 >= found:119","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911937478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911950126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911962565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"":"downstairs","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911975243Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911986711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"0","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.911999479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"1","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912012117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"client":"2","":"downstairs","session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912024945Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912036814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"95717424-86dc-4fb2-bab2-ab1a197aff04 is now active with session: 6e5754f9-c6b1-4e46-9a11-390887185c4b"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912056331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912068219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662,"session_id":"6e5754f9-c6b1-4e46-9a11-390887185c4b"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:25.912080177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 571 len 16384 data: 11 11 12 13 02/30 Read block 5 len 8192 03/30 Read block 528 len 4096 04/30 Write block 329 len 32768 data: 17 14 12 11 9 9 11 10 05/30 Read block 465 len 16384 06/30 Read block 369 len 32768 07/30 Read block 95 len 16384 08/30 Read block 410 len 8192 09/30 Read block 32 len 32768 10/30 Read block 120 len 36864 11/30 Write block 10 len 4096 data: 8 12/30 Write block 444 len 8192 data: 10 9 13/30 Write block 518 len 4096 data: 11 14/30 Write block 61 len 24576 data: 5 9 13 12 15 12 15/30 Write block 306 len 4096 data: 6 16/30 Write block 418 len 32768 data: 11 13 13 12 13 10 10 11 17/30 Read block 390 len 32768 18/30 Read block 30 len 24576 19/30 Write block 276 len 8192 data: 11 12 20/30 Read block 386 len 32768 21/30 Write block 480 len 24576 data: 9 9 8 7 7 7 22/30 Read block 247 len 12288 23/30 Write block 429 len 32768 data: 11 13 12 10 9 9 13 11 24/30 Write block 556 len 32768 data: 12 10 11 11 8 7 8 8 25/30 Write block 330 len 4096 data: 15 26/30 Read block 590 len 4096 27/30 Read block 18 len 40960 28/30 Write block 589 len 12288 data: 8 7 7 29/30 Read block 571 len 24576 30/30 Write block 312 len 20480 data: 9 9 8 8 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:25.929835563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 T T T 1 020-039 118 118 118 190 190 190 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 100 100 100 161 161 161 T T T 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 118 118 118 190 190 190 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 118 118 118 190 190 190 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 104 104 104 171 171 171 T T T 14 280-299 105 105 105 172 172 172 F F F 15 300-319 101 101 101 166 166 166 T T T 16 320-339 118 118 118 190 190 190 T T T 17 340-359 118 118 118 190 190 190 F F F 18 360-379 118 118 118 190 190 190 F F F 19 380-399 116 116 116 188 188 188 F F F 20 400-419 115 115 115 185 185 185 T T T 21 420-439 118 118 118 190 190 190 T T T 22 440-459 118 118 118 190 190 190 T T T 23 460-479 118 118 118 190 190 190 F F F 24 480-499 116 116 116 188 188 188 T T T 25 500-519 118 118 118 190 190 190 T T T 26 520-539 118 118 118 190 190 190 F F F 27 540-559 118 118 118 190 190 190 T T T 28 560-579 115 115 115 185 185 185 T T T 29 580-599 117 117 117 189 189 189 T T T Max gen: 118, Max flush: 190 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 118 118 118 190 190 190 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 118 118 118 190 190 190 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 118 118 118 190 190 190 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 119 119 119 191 191 191 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 119 119 119 191 191 191 F F F 16 320-339 119 119 119 191 191 191 F F F 17 340-359 118 118 118 190 190 190 F F F 18 360-379 118 118 118 190 190 190 F F F 19 380-399 116 116 116 188 188 188 F F F 20 400-419 119 119 119 191 191 191 F F F 21 420-439 119 119 119 191 191 191 F F F 22 440-459 119 119 119 191 191 191 F F F 23 460-479 118 118 118 190 190 190 F F F 24 480-499 119 119 119 191 191 191 F F F 25 500-519 119 119 119 191 191 191 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 119 119 119 191 191 191 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 119, Max flush: 191 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.214334757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:30.258240877Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.258598545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.291605681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.292182377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.292216632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.29223009Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.292244578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} {"msg":"Crucible 6269e5f0-0b23-42ac-affa-2d42731a5769 has session id: 52cdb3e3-d504-41d7-8c63-85adc9fa4f0a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.292747255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Upstairs opts: Upstairs UUID: 6269e5f0-0b23-42ac-affa-2d42731a5769, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.29278348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Crucible stats registered with UUID: 6269e5f0-0b23-42ac-affa-2d42731a5769","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.292800007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:30.292813856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293215427Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293240933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} {"msg":"6269e5f0-0b23-42ac-affa-2d42731a5769 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293305234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293323851Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293346998Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293361046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293378883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293393071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293416508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293439034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293461801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293484698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36868)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293508564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51640)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293534121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64014)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.293559707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.294240938Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.294273473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.294310088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.294848429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.294877545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.29491291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295408968Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295433804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[0]R flush_numbers[0..12]: [191, 190, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295447452Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[0]R generation[0..12]: [119, 118, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.29546178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295476018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[1]R flush_numbers[0..12]: [191, 190, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295489646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[1]R generation[0..12]: [119, 118, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295503064Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295516732Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[2]R flush_numbers[0..12]: [191, 190, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.2955302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[2]R generation[0..12]: [119, 118, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295543538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295578043Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295590911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Generation requested: 122 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295603779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295619827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"All extents match","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295639004Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"":"downstairs","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295651992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295663511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"0","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295676519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"1","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295689087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"client":"2","":"downstairs","session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295701915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295713913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"6269e5f0-0b23-42ac-affa-2d42731a5769 is now active with session: 52cdb3e3-d504-41d7-8c63-85adc9fa4f0a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295726272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.2957379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670,"session_id":"52cdb3e3-d504-41d7-8c63-85adc9fa4f0a"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:30.295749748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 28672 data: 8 8 8 7 6 7 8 02/30 Read block 177 len 24576 03/30 Write block 376 len 32768 data: 9 11 11 10 10 10 10 9 04/30 Flush 05/30 Write block 344 len 20480 data: 12 13 13 13 12 06/30 Write block 404 len 32768 data: 17 17 15 14 13 13 13 10 07/30 Read block 555 len 32768 08/30 Read block 238 len 4096 09/30 Read block 302 len 20480 10/30 Read block 79 len 4096 11/30 Flush 12/30 Flush 13/30 Write block 408 len 36864 data: 14 14 14 11 13 11 12 12 11 14/30 Read block 528 len 24576 15/30 Read block 588 len 8192 16/30 Write block 364 len 24576 data: 9 9 10 10 9 9 17/30 Write block 418 len 28672 data: 12 14 14 13 14 11 11 18/30 Read block 337 len 12288 19/30 Write block 6 len 28672 data: 6 8 7 7 9 6 7 20/30 Read block 217 len 16384 21/30 Write block 569 len 12288 data: 9 10 12 22/30 Read block 451 len 36864 23/30 Write block 447 len 8192 data: 10 8 24/30 Read block 174 len 20480 25/30 Write block 568 len 24576 data: 9 10 11 13 12 13 26/30 Read block 124 len 28672 27/30 Write block 23 len 16384 data: 5 6 6 6 28/30 Read block 455 len 28672 29/30 Read block 498 len 12288 30/30 Write block 4 len 32768 data: 8 7 7 9 8 8 10 7 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:30.31628058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 T T T 1 020-039 118 118 118 190 190 190 T T T 2 040-059 114 114 114 183 183 183 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 118 118 118 190 190 190 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 118 118 118 190 190 190 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 119 119 119 191 191 191 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 119 119 119 191 191 191 F F F 16 320-339 119 119 119 191 191 191 F F F 17 340-359 122 122 122 193 193 193 F F F 18 360-379 122 122 122 192 192 192 T T T 19 380-399 122 122 122 192 192 192 F F F 20 400-419 122 122 122 193 193 193 T T T 21 420-439 119 119 119 191 191 191 T T T 22 440-459 119 119 119 191 191 191 T T T 23 460-479 118 118 118 190 190 190 F F F 24 480-499 122 122 122 192 192 192 F F F 25 500-519 119 119 119 191 191 191 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 119 119 119 191 191 191 T T T 29 580-599 119 119 119 191 191 191 F F F Max gen: 122, Max flush: 193 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 194 194 194 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 114 114 114 183 183 183 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 110 110 110 178 178 178 F F F 7 140-159 118 118 118 190 190 190 F F F 8 160-179 104 104 104 171 171 171 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 118 118 118 190 190 190 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 119 119 119 191 191 191 F F F 14 280-299 105 105 105 172 172 172 F F F 15 300-319 119 119 119 191 191 191 F F F 16 320-339 119 119 119 191 191 191 F F F 17 340-359 122 122 122 193 193 193 F F F 18 360-379 123 123 123 194 194 194 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 123 123 123 194 194 194 F F F 22 440-459 123 123 123 194 194 194 F F F 23 460-479 118 118 118 190 190 190 F F F 24 480-499 122 122 122 192 192 192 F F F 25 500-519 119 119 119 191 191 191 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 123, Max flush: 194 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.600295415Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.645019136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.645658733Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.645703446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.645720044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.645734572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} {"msg":"Crucible 5c5efc7d-e552-44f4-af61-1993dafb6871 has session id: ab828ae9-0167-42e6-aa9b-ccb92c5220d5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.646462796Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Upstairs opts: Upstairs UUID: 5c5efc7d-e552-44f4-af61-1993dafb6871, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.646498221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Crucible stats registered with UUID: 5c5efc7d-e552-44f4-af61-1993dafb6871","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.646518578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:34.646531976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.64698219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647003647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} {"msg":"5c5efc7d-e552-44f4-af61-1993dafb6871 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647077926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647100313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647137398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647163484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.64719128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647214936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647240363Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647268039Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647296805Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647312222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37240)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647336579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:50493)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647367494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:55349)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647499505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.647999262Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.648037837Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.648452066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.648583627Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.64876742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649132697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649247821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649277846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[0]R flush_numbers[0..12]: [194, 194, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649612348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[0]R generation[0..12]: [123, 123, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649638644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649654251Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[1]R flush_numbers[0..12]: [194, 194, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649668519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[1]R generation[0..12]: [123, 123, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649685937Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649700475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[2]R flush_numbers[0..12]: [194, 194, 183, 191, 190, 189, 178, 190, 171, 189, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649714493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[2]R generation[0..12]: [123, 123, 114, 119, 118, 117, 110, 118, 104, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649727951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649741599Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649754257Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649777723Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.64980165Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649825726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"":"downstairs","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649849243Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.64987062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"0","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649896586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"1","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649920992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"client":"2","":"downstairs","session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649948808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649963896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"5c5efc7d-e552-44f4-af61-1993dafb6871 is now active with session: ab828ae9-0167-42e6-aa9b-ccb92c5220d5","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649979614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.649991602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678,"session_id":"ab828ae9-0167-42e6-aa9b-ccb92c5220d5"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:34.650012979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 484 len 16384 data: 9 9 8 7 02/30 Write block 321 len 8192 data: 5 7 03/30 Read block 517 len 40960 04/30 Write block 171 len 36864 data: 7 7 10 8 7 8 8 8 8 05/30 Read block 95 len 4096 06/30 Write block 293 len 24576 data: 9 12 14 13 10 8 07/30 Flush 08/30 Write block 465 len 28672 data: 11 11 10 12 11 11 9 09/30 Flush 10/30 Write block 209 len 16384 data: 10 9 9 8 11/30 Write block 148 len 32768 data: 16 19 14 13 10 9 8 8 12/30 Read block 19 len 28672 13/30 Read block 319 len 36864 14/30 Write block 431 len 24576 data: 13 11 10 10 14 12 15/30 Read block 394 len 32768 16/30 Read block 449 len 24576 17/30 Write block 129 len 24576 data: 6 7 7 8 8 9 18/30 Write block 46 len 32768 data: 10 11 9 9 8 10 10 9 19/30 Read block 369 len 24576 20/30 Read block 526 len 32768 21/30 Read block 107 len 8192 22/30 Flush 23/30 Write block 300 len 32768 data: 8 8 7 7 9 6 7 6 24/30 Read block 86 len 36864 25/30 Read block 42 len 32768 26/30 Write block 342 len 28672 data: 13 11 13 14 14 14 13 27/30 Write block 290 len 16384 data: 12 10 9 10 28/30 Read block 480 len 8192 29/30 Write block 329 len 24576 data: 18 16 13 12 10 10 30/30 Read block 119 len 12288 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:34.670297057Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 194 194 194 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 124 124 124 197 197 197 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 124 124 124 197 197 197 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 124 124 124 195 195 195 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 124 124 124 197 197 197 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 119 119 119 191 191 191 F F F 14 280-299 124 124 124 195 195 195 T T T 15 300-319 119 119 119 191 191 191 T T T 16 320-339 124 124 124 195 195 195 T T T 17 340-359 122 122 122 193 193 193 T T T 18 360-379 123 123 123 194 194 194 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 123 123 123 194 194 194 F F F 23 460-479 124 124 124 196 196 196 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 119 119 119 191 191 191 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 124, Max flush: 197 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 194 194 194 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 124 124 124 197 197 197 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 124 124 124 197 197 197 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 124 124 124 195 195 195 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 124 124 124 197 197 197 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 119 119 119 191 191 191 F F F 14 280-299 125 125 125 198 198 198 F F F 15 300-319 125 125 125 198 198 198 F F F 16 320-339 125 125 125 198 198 198 F F F 17 340-359 125 125 125 198 198 198 F F F 18 360-379 123 123 123 194 194 194 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 123 123 123 194 194 194 F F F 23 460-479 124 124 124 196 196 196 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 119 119 119 191 191 191 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 125, Max flush: 198 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.898120379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.942043756Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94270787Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.942754473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94277664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.942791738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} {"msg":"Crucible 1e0a01f5-13e8-4d25-8a84-b2545294d583 has session id: c7502e07-a683-42a0-9f27-77f31c8f45c8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.943432994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Upstairs opts: Upstairs UUID: 1e0a01f5-13e8-4d25-8a84-b2545294d583, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.943478598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Crucible stats registered with UUID: 1e0a01f5-13e8-4d25-8a84-b2545294d583","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.943503534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:38.943526001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944003551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944040316Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} {"msg":"1e0a01f5-13e8-4d25-8a84-b2545294d583 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944095228Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944121704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94414874Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944164508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944179096Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944192414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944210521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944234438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944262364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94428844Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62102)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944316296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52141)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944608663Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52086)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.944653007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.945171661Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.945201137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.946012969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.946057992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.946592924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.946756131Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.946781387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947073854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[0]R flush_numbers[0..12]: [194, 194, 197, 191, 190, 189, 197, 197, 195, 189, 197, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947571862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[0]R generation[0..12]: [123, 123, 124, 119, 118, 117, 124, 124, 124, 117, 124, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947592679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947607727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[1]R flush_numbers[0..12]: [194, 194, 197, 191, 190, 189, 197, 197, 195, 189, 197, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947621505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[1]R generation[0..12]: [123, 123, 124, 119, 118, 117, 124, 124, 124, 117, 124, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947634913Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947649121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[2]R flush_numbers[0..12]: [194, 194, 197, 191, 190, 189, 197, 197, 195, 189, 197, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947662739Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[2]R generation[0..12]: [123, 123, 124, 119, 118, 117, 124, 124, 124, 117, 124, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947676247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947689765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947702333Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947715151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947727929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947740467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":Wait for a query_work_queue command to finish before sending IO 1686,"":"downstairs","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947763474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947775682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"0","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94778847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"1","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947801349Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"client":"2","":"downstairs","session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947814457Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947826465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"1e0a01f5-13e8-4d25-8a84-b2545294d583 is now active with session: c7502e07-a683-42a0-9f27-77f31c8f45c8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947838873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.947850491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686,"session_id":"c7502e07-a683-42a0-9f27-77f31c8f45c8"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:38.94786262Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 4096 data: 13 02/30 Write block 200 len 4096 data: 9 03/30 Write block 8 len 16384 data: 9 9 11 8 04/30 Write block 307 len 12288 data: 7 7 8 05/30 Write block 280 len 4096 data: 8 06/30 Write block 276 len 40960 data: 12 13 11 8 9 12 11 11 14 16 07/30 Read block 416 len 28672 08/30 Flush 09/30 Write block 441 len 40960 data: 8 9 11 11 10 9 11 9 8 7 10/30 Read block 430 len 8192 11/30 Read block 213 len 8192 12/30 Read block 479 len 4096 13/30 Write block 200 len 12288 data: 10 9 10 14/30 Write block 443 len 8192 data: 12 12 15/30 Flush 16/30 Write block 459 len 16384 data: 10 11 9 12 17/30 Flush 18/30 Write block 60 len 28672 data: 5 6 10 14 13 16 13 19/30 Write block 209 len 4096 data: 11 20/30 Read block 334 len 28672 21/30 Read block 277 len 4096 22/30 Write block 133 len 20480 data: 9 10 6 6 7 23/30 Write block 503 len 4096 data: 16 24/30 Write block 152 len 40960 data: 11 10 9 9 10 9 7 6 7 9 25/30 Flush 26/30 Write block 279 len 8192 data: 9 10 27/30 Write block 129 len 32768 data: 7 8 8 9 10 11 7 7 28/30 Write block 507 len 28672 data: 19 19 15 12 10 9 8 29/30 Read block 30 len 16384 30/30 Read block 318 len 28672 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:38.965867759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 124 124 124 197 197 197 F F F 3 060-079 126 126 126 202 202 202 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 126 126 126 202 202 202 T T T 7 140-159 126 126 126 202 202 202 F F F 8 160-179 126 126 126 202 202 202 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 126 126 126 202 202 202 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 126 126 126 199 199 199 T T T 14 280-299 126 126 126 199 199 199 T T T 15 300-319 126 126 126 199 199 199 F F F 16 320-339 125 125 125 198 198 198 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 123 123 123 194 194 194 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 126 126 126 201 201 201 F F F 23 460-479 126 126 126 201 201 201 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 126 126 126 202 202 202 T T T 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 126, Max flush: 202 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 124 124 124 197 197 197 F F F 3 060-079 126 126 126 202 202 202 F F F 4 080-099 118 118 118 190 190 190 F F F 5 100-119 117 117 117 189 189 189 F F F 6 120-139 127 127 127 203 203 203 F F F 7 140-159 126 126 126 202 202 202 F F F 8 160-179 126 126 126 202 202 202 F F F 9 180-199 117 117 117 189 189 189 F F F 10 200-219 126 126 126 202 202 202 F F F 11 220-239 118 118 118 190 190 190 F F F 12 240-259 116 116 116 188 188 188 F F F 13 260-279 127 127 127 203 203 203 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 126 126 126 199 199 199 F F F 16 320-339 125 125 125 198 198 198 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 123 123 123 194 194 194 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 126 126 126 201 201 201 F F F 23 460-479 126 126 126 201 201 201 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 127, Max flush: 203 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.210800602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.255344449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256065974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256110897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256126375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256143423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} {"msg":"Crucible 8cd581b5-5fd2-404f-ba5b-10b8bebd6de8 has session id: 0dcae2e4-3cab-464c-a2cb-e8491659eb3d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256755084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Upstairs opts: Upstairs UUID: 8cd581b5-5fd2-404f-ba5b-10b8bebd6de8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256809726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Crucible stats registered with UUID: 8cd581b5-5fd2-404f-ba5b-10b8bebd6de8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.256826853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:43.256839761Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257291026Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257316272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} {"msg":"8cd581b5-5fd2-404f-ba5b-10b8bebd6de8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257376863Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257408198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257433865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257448753Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.2574684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257530951Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257557197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257571445Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257584883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257598351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62746)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257611979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49351)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257846925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55510)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.257889099Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.258309158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.258339043Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.25835824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.258965982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.258994568Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259010005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.25952975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259647793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[0]R flush_numbers[0..12]: [199, 194, 197, 202, 190, 189, 203, 202, 202, 189, 202, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259716563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[0]R generation[0..12]: [126, 123, 124, 126, 118, 117, 127, 126, 126, 117, 126, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.2597367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259751368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[1]R flush_numbers[0..12]: [199, 194, 197, 202, 190, 189, 203, 202, 202, 189, 202, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259765095Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[1]R generation[0..12]: [126, 123, 124, 126, 118, 117, 127, 126, 126, 117, 126, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259778504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259792112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[2]R flush_numbers[0..12]: [199, 194, 197, 202, 190, 189, 203, 202, 202, 189, 202, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.25980555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[2]R generation[0..12]: [126, 123, 124, 126, 118, 117, 127, 126, 126, 117, 126, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259819018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259832886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259849803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:15:43.259863071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259879029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259891697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"":"downstairs","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259904375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259916084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"0","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259929192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"1","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.25994198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"client":"2","":"downstairs","session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259955168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259967246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"8cd581b5-5fd2-404f-ba5b-10b8bebd6de8 is now active with session: 0dcae2e4-3cab-464c-a2cb-e8491659eb3d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259979864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.259991403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694,"session_id":"0dcae2e4-3cab-464c-a2cb-e8491659eb3d"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:43.260003241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 87 len 20480 data: 6 6 7 6 6 02/30 Write block 331 len 16384 data: 14 13 11 11 03/30 Write block 365 len 32768 data: 10 11 11 10 10 9 9 7 04/30 Read block 366 len 40960 05/30 Read block 445 len 16384 06/30 Write block 265 len 16384 data: 9 10 8 6 07/30 Flush 08/30 Write block 331 len 16384 data: 15 14 12 12 09/30 Read block 521 len 4096 10/30 Write block 56 len 12288 data: 7 9 8 11/30 Write block 312 len 8192 data: 10 10 12/30 Write block 189 len 20480 data: 14 13 13 11 11 13/30 Read block 130 len 20480 14/30 Write block 209 len 4096 data: 12 15/30 Write block 238 len 36864 data: 14 15 16 13 15 12 9 9 8 16/30 Flush 17/30 Write block 97 len 24576 data: 12 13 14 14 14 11 18/30 Read block 450 len 16384 19/30 Read block 154 len 40960 20/30 Write block 169 len 12288 data: 9 7 8 21/30 Write block 261 len 24576 data: 6 6 7 8 10 11 22/30 Read block 498 len 28672 23/30 Read block 290 len 32768 24/30 Flush 25/30 Write block 447 len 20480 data: 12 10 9 8 9 26/30 Read block 88 len 40960 27/30 Write block 69 len 36864 data: 11 10 8 8 8 7 8 7 8 28/30 Read block 110 len 32768 29/30 Read block 255 len 40960 30/30 Write block 264 len 36864 data: 9 11 12 9 7 7 7 7 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:43.27952562Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 128 128 128 205 205 205 F F F 3 060-079 126 126 126 202 202 202 T T T 4 080-099 128 128 128 206 206 206 F F F 5 100-119 128 128 128 206 206 206 F F F 6 120-139 127 127 127 203 203 203 F F F 7 140-159 126 126 126 202 202 202 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 128 128 128 205 205 205 F F F 10 200-219 128 128 128 205 205 205 F F F 11 220-239 128 128 128 205 205 205 F F F 12 240-259 128 128 128 205 205 205 F F F 13 260-279 128 128 128 206 206 206 T T T 14 280-299 127 127 127 203 203 203 F F F 15 300-319 128 128 128 205 205 205 F F F 16 320-339 128 128 128 205 205 205 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 126 126 126 201 201 201 T T T 23 460-479 126 126 126 201 201 201 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 128, Max flush: 206 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 123 123 123 194 194 194 F F F 2 040-059 128 128 128 205 205 205 F F F 3 060-079 129 129 129 207 207 207 F F F 4 080-099 128 128 128 206 206 206 F F F 5 100-119 128 128 128 206 206 206 F F F 6 120-139 127 127 127 203 203 203 F F F 7 140-159 126 126 126 202 202 202 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 128 128 128 205 205 205 F F F 10 200-219 128 128 128 205 205 205 F F F 11 220-239 128 128 128 205 205 205 F F F 12 240-259 128 128 128 205 205 205 F F F 13 260-279 129 129 129 207 207 207 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 128 128 128 205 205 205 F F F 16 320-339 128 128 128 205 205 205 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 129 129 129 207 207 207 F F F 23 460-479 126 126 126 201 201 201 F F F 24 480-499 124 124 124 195 195 195 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 129, Max flush: 207 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.506979606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.550435382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551015577Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551055511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551070219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551084497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} {"msg":"Crucible 639e3cde-13ca-48f9-8130-19a37e8ae50f has session id: 1268f99a-45a1-4bf2-a628-b038d785b276","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551745031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Upstairs opts: Upstairs UUID: 639e3cde-13ca-48f9-8130-19a37e8ae50f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551787255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Crucible stats registered with UUID: 639e3cde-13ca-48f9-8130-19a37e8ae50f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.551815371Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:47.551836758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552317798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552343034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} {"msg":"639e3cde-13ca-48f9-8130-19a37e8ae50f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552396896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552422382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552445609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552460477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552519298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552554453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552594307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552621024Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.55264557Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552670306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"ds_connection connected from Ok(127.0.0.1:39738)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552694573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"ds_connection connected from Ok(127.0.0.1:65192)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552859059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"ds_connection connected from Ok(127.0.0.1:61851)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.552890694Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.553284577Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.553466141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.553893318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.554029359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.554434569Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.554503659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.554613634Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.554974491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[0]R flush_numbers[0..12]: [199, 194, 205, 207, 206, 206, 203, 202, 206, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555457231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[0]R generation[0..12]: [126, 123, 128, 129, 128, 128, 127, 126, 128, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555481597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555496365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[1]R flush_numbers[0..12]: [199, 194, 205, 207, 206, 206, 203, 202, 206, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555510273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[1]R generation[0..12]: [126, 123, 128, 129, 128, 128, 127, 126, 128, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555527341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555541188Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[2]R flush_numbers[0..12]: [199, 194, 205, 207, 206, 206, 203, 202, 206, 205, 205, 205]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555554477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[2]R generation[0..12]: [126, 123, 128, 129, 128, 128, 127, 126, 128, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555567655Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555581113Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555593681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555609849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555633005Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555659621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"":"downstairs","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555673809Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555685347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"0","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555702645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"1","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555725662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"client":"2","":"downstairs","session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555749138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555771505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"639e3cde-13ca-48f9-8130-19a37e8ae50f is now active with session: 1268f99a-45a1-4bf2-a628-b038d785b276","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555798191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555821618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702,"session_id":"1268f99a-45a1-4bf2-a628-b038d785b276"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:47.555844294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 12288 data: 14 14 19 02/30 Read block 318 len 40960 03/30 Read block 7 len 32768 04/30 Write block 40 len 40960 data: 15 11 9 9 6 9 11 12 10 10 05/30 Write block 450 len 8192 data: 9 10 06/30 Write block 33 len 16384 data: 12 12 10 10 07/30 Read block 375 len 32768 08/30 Write block 75 len 16384 data: 9 8 9 7 09/30 Read block 58 len 32768 10/30 Read block 58 len 20480 11/30 Write block 229 len 20480 data: 8 9 10 14 11 12/30 Read block 25 len 40960 13/30 Write block 220 len 20480 data: 7 8 8 8 7 14/30 Flush 15/30 Write block 249 len 8192 data: 7 6 16/30 Write block 241 len 36864 data: 14 16 13 10 10 9 8 7 8 17/30 Write block 318 len 8192 data: 7 6 18/30 Write block 489 len 8192 data: 9 7 19/30 Read block 30 len 12288 20/30 Write block 190 len 36864 data: 14 14 12 12 10 10 11 12 13 21/30 Flush 22/30 Write block 106 len 32768 data: 9 13 12 10 8 9 9 9 23/30 Write block 133 len 40960 data: 11 12 8 8 8 6 6 5 8 10 24/30 Write block 564 len 20480 data: 6 4 6 8 10 25/30 Write block 61 len 20480 data: 7 11 15 14 17 26/30 Write block 205 len 24576 data: 7 7 7 10 13 10 27/30 Read block 38 len 16384 28/30 Write block 323 len 40960 data: 9 12 13 13 15 15 20 17 16 15 29/30 Write block 267 len 40960 data: 10 8 8 8 8 11 10 11 12 13 30/30 Write block 71 len 24576 data: 9 9 9 8 10 9 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:47.574455286Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 130 130 130 208 208 208 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 130 130 130 208 208 208 T T T 4 080-099 128 128 128 206 206 206 F F F 5 100-119 128 128 128 206 206 206 T T T 6 120-139 127 127 127 203 203 203 T T T 7 140-159 126 126 126 202 202 202 T T T 8 160-179 128 128 128 206 206 206 F F F 9 180-199 130 130 130 209 209 209 F F F 10 200-219 128 128 128 205 205 205 T T T 11 220-239 130 130 130 208 208 208 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 129 129 129 207 207 207 T T T 14 280-299 127 127 127 203 203 203 F F F 15 300-319 130 130 130 209 209 209 F F F 16 320-339 130 130 130 208 208 208 T T T 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 130 130 130 208 208 208 F F F 23 460-479 126 126 126 201 201 201 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 123 123 123 194 194 194 T T T 29 580-599 119 119 119 191 191 191 F F F Max gen: 130, Max flush: 209 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 130 130 130 208 208 208 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 131 131 131 210 210 210 F F F 4 080-099 128 128 128 206 206 206 F F F 5 100-119 131 131 131 210 210 210 F F F 6 120-139 131 131 131 210 210 210 F F F 7 140-159 131 131 131 210 210 210 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 130 130 130 209 209 209 F F F 10 200-219 131 131 131 210 210 210 F F F 11 220-239 130 130 130 208 208 208 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 131 131 131 210 210 210 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 130 130 130 209 209 209 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 122 122 122 192 192 192 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 130 130 130 208 208 208 F F F 23 460-479 126 126 126 201 201 201 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 131 131 131 210 210 210 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 131, Max flush: 210 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.831823758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:51.874528853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.874881061Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.906660296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.907243341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.907285435Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.907302603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.90732122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} {"msg":"Crucible 588bdfe7-46ef-40da-89ff-af78ffa5d486 has session id: 10f1ce9f-f65c-43d9-a309-1831032bc347","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.907998621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Upstairs opts: Upstairs UUID: 588bdfe7-46ef-40da-89ff-af78ffa5d486, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908046384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Crucible stats registered with UUID: 588bdfe7-46ef-40da-89ff-af78ffa5d486","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908071431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:51.908089498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908475772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908499588Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} {"msg":"588bdfe7-46ef-40da-89ff-af78ffa5d486 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908565029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908581916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908604673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908622651Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908637428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908651136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908665684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908678612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.90869385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908718226Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"ds_connection connected from Ok(127.0.0.1:41432)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908807453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"ds_connection connected from Ok(127.0.0.1:40162)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908843688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"ds_connection connected from Ok(127.0.0.1:36391)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.908869384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.909551935Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.909592419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.909717141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.910243844Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.91027645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.910379625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.910862425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.910981867Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[0]R flush_numbers[0..12]: [199, 208, 208, 210, 206, 210, 210, 210, 206, 209, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911013862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[0]R generation[0..12]: [126, 130, 130, 131, 128, 131, 131, 131, 128, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.9110299Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911044158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[1]R flush_numbers[0..12]: [199, 208, 208, 210, 206, 210, 210, 210, 206, 209, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911057936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[1]R generation[0..12]: [126, 130, 130, 131, 128, 131, 131, 131, 128, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911071804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911085532Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[2]R flush_numbers[0..12]: [199, 208, 208, 210, 206, 210, 210, 210, 206, 209, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.91109878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[2]R generation[0..12]: [126, 130, 130, 131, 128, 131, 131, 131, 128, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911112238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911126046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911138534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911154862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.91116768Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911180468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"":"downstairs","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911209244Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:15:51.911223342Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"0","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911243269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"1","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911255897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"client":"2","":"downstairs","session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911268665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911280644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"588bdfe7-46ef-40da-89ff-af78ffa5d486 is now active with session: 10f1ce9f-f65c-43d9-a309-1831032bc347","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911293112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.91130469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710,"session_id":"10f1ce9f-f65c-43d9-a309-1831032bc347"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:51.911316618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 24576 data: 10 12 13 11 9 9 02/30 Read block 369 len 16384 03/30 Read block 35 len 8192 04/30 Flush 05/30 Write block 381 len 24576 data: 11 11 10 7 9 9 06/30 Flush 07/30 Flush 08/30 Flush 09/30 Write block 61 len 16384 data: 8 12 16 15 10/30 Read block 437 len 16384 11/30 Flush 12/30 Write block 476 len 8192 data: 12 12 13/30 Read block 274 len 16384 14/30 Read block 268 len 12288 15/30 Read block 150 len 32768 16/30 Write block 26 len 8192 data: 7 6 17/30 Read block 334 len 8192 18/30 Read block 543 len 36864 19/30 Write block 77 len 16384 data: 10 8 8 7 20/30 Write block 191 len 12288 data: 15 13 13 21/30 Read block 95 len 28672 22/30 Write block 107 len 20480 data: 14 13 11 9 10 23/30 Flush 24/30 Write block 310 len 36864 data: 7 7 11 11 9 9 8 9 8 25/30 Read block 478 len 12288 26/30 Write block 460 len 4096 data: 12 27/30 Read block 410 len 40960 28/30 Read block 254 len 28672 29/30 Read block 73 len 16384 30/30 Read block 318 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:51.929760344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 132 132 132 216 216 216 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 132 132 132 216 216 216 F F F 4 080-099 132 132 132 216 216 216 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 131 131 131 210 210 210 F F F 7 140-159 131 131 131 210 210 210 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 132 132 132 216 216 216 F F F 10 200-219 131 131 131 210 210 210 F F F 11 220-239 130 130 130 208 208 208 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 132 132 132 211 211 211 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 130 130 130 209 209 209 T T T 16 320-339 131 131 131 210 210 210 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 132 132 132 212 212 212 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 130 130 130 208 208 208 F F F 23 460-479 132 132 132 216 216 216 T T T 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 131 131 131 210 210 210 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 132, Max flush: 216 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 F F F 1 020-039 132 132 132 216 216 216 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 132 132 132 216 216 216 F F F 4 080-099 132 132 132 216 216 216 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 131 131 131 210 210 210 F F F 7 140-159 131 131 131 210 210 210 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 132 132 132 216 216 216 F F F 10 200-219 131 131 131 210 210 210 F F F 11 220-239 130 130 130 208 208 208 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 132 132 132 211 211 211 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 132 132 132 212 212 212 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 F F F 22 440-459 130 130 130 208 208 208 F F F 23 460-479 133 133 133 217 217 217 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 131 131 131 210 210 210 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 133, Max flush: 217 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.188874683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.234616006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235240025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235283598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235298487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235312594Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} {"msg":"Crucible 0e9ff674-4698-4ed8-8e78-97d55b55eb93 has session id: ee17440e-e0a9-4ac2-8098-3606b85780c2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235907888Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Upstairs opts: Upstairs UUID: 0e9ff674-4698-4ed8-8e78-97d55b55eb93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235941543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Crucible stats registered with UUID: 0e9ff674-4698-4ed8-8e78-97d55b55eb93","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.235956091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:56.235968709Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236395497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236418224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} {"msg":"0e9ff674-4698-4ed8-8e78-97d55b55eb93 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236479675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236496182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236517569Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236531117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236544955Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236557813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236571561Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236584279Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236601267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236625383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36004)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236659368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47211)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.236688574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45175)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.23671411Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.237408639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.237441335Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.237457372Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.237997094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238168719Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.23822701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238688123Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238795018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[0]R flush_numbers[0..12]: [199, 216, 208, 216, 216, 216, 210, 210, 206, 216, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238905302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[0]R generation[0..12]: [126, 132, 130, 132, 132, 132, 131, 131, 128, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238928098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238943676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[1]R flush_numbers[0..12]: [199, 216, 208, 216, 216, 216, 210, 210, 206, 216, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238957924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[1]R generation[0..12]: [126, 132, 130, 132, 132, 132, 131, 131, 128, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.238971672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.23898526Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[2]R flush_numbers[0..12]: [199, 216, 208, 216, 216, 216, 210, 210, 206, 216, 210, 208]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239011966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[2]R generation[0..12]: [126, 132, 130, 132, 132, 132, 131, 131, 128, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239039252Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.23905592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239069268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:15:56.239082306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239099803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239112671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"":"downstairs","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239125559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239137498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"0","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239150716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"1","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239163584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"client":"2","":"downstairs","session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239176372Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.23918854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"0e9ff674-4698-4ed8-8e78-97d55b55eb93 is now active with session: ee17440e-e0a9-4ac2-8098-3606b85780c2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239201798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239213547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718,"session_id":"ee17440e-e0a9-4ac2-8098-3606b85780c2"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:15:56.239225445Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 440 len 24576 data: 7 9 10 13 13 11 02/30 Write block 471 len 28672 data: 10 12 11 11 13 13 13 03/30 Flush 04/30 Write block 190 len 28672 data: 15 16 14 14 11 11 12 05/30 Write block 272 len 16384 data: 12 11 12 13 06/30 Read block 116 len 16384 07/30 Write block 14 len 32768 data: 6 6 3 3 5 5 6 3 08/30 Write block 533 len 20480 data: 10 10 13 13 15 09/30 Write block 135 len 32768 data: 9 9 9 7 7 6 9 11 10/30 Write block 80 len 16384 data: 8 5 5 6 11/30 Read block 366 len 24576 12/30 Read block 48 len 28672 13/30 Read block 213 len 40960 14/30 Write block 565 len 36864 data: 5 7 9 11 11 12 14 13 14 15/30 Read block 584 len 28672 16/30 Read block 373 len 36864 17/30 Write block 545 len 32768 data: 10 9 11 12 11 8 9 7 18/30 Read block 455 len 4096 19/30 Read block 364 len 36864 20/30 Read block 349 len 8192 21/30 Read block 378 len 24576 22/30 Read block 456 len 4096 23/30 Write block 222 len 24576 data: 9 9 8 7 10 9 24/30 Read block 447 len 4096 25/30 Read block 559 len 4096 26/30 Read block 150 len 36864 27/30 Write block 421 len 24576 data: 14 15 12 12 12 8 28/30 Read block 468 len 40960 29/30 Write block 14 len 20480 data: 7 7 4 4 6 30/30 Write block 388 len 8192 data: 13 14 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:15:56.258535354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 199 199 199 T T T 1 020-039 132 132 132 216 216 216 T T T 2 040-059 130 130 130 208 208 208 F F F 3 060-079 132 132 132 216 216 216 F F F 4 080-099 132 132 132 216 216 216 T T T 5 100-119 132 132 132 216 216 216 F F F 6 120-139 131 131 131 210 210 210 T T T 7 140-159 131 131 131 210 210 210 T T T 8 160-179 128 128 128 206 206 206 F F F 9 180-199 132 132 132 216 216 216 T T T 10 200-219 131 131 131 210 210 210 F F F 11 220-239 130 130 130 208 208 208 T T T 12 240-259 130 130 130 209 209 209 F F F 13 260-279 132 132 132 211 211 211 T T T 14 280-299 127 127 127 203 203 203 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 132 132 132 212 212 212 T T T 20 400-419 123 123 123 194 194 194 F F F 21 420-439 124 124 124 197 197 197 T T T 22 440-459 134 134 134 218 218 218 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 118 118 118 190 190 190 T T T 27 540-559 119 119 119 191 191 191 T T T 28 560-579 131 131 131 210 210 210 T T T 29 580-599 119 119 119 191 191 191 F F F Max gen: 134, Max flush: 218 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 219 219 219 F F F 1 020-039 135 135 135 219 219 219 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 132 132 132 216 216 216 F F F 4 080-099 135 135 135 219 219 219 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 135 135 135 219 219 219 F F F 7 140-159 135 135 135 219 219 219 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 135 135 135 219 219 219 F F F 10 200-219 131 131 131 210 210 210 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 135 135 135 219 219 219 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 126 126 126 199 199 199 F F F 18 360-379 128 128 128 204 204 204 F F F 19 380-399 135 135 135 219 219 219 F F F 20 400-419 123 123 123 194 194 194 F F F 21 420-439 135 135 135 219 219 219 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 135 135 135 219 219 219 F F F 28 560-579 135 135 135 219 219 219 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 135, Max flush: 219 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.53738785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:00.58006109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.580456592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.612433208Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.612995216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613029621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613043319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613057847Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} {"msg":"Crucible 60e6427a-79a0-425c-9d72-2a130236b39e has session id: d5b51487-e577-4cad-bca6-6aaa045d5433","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613649541Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Upstairs opts: Upstairs UUID: 60e6427a-79a0-425c-9d72-2a130236b39e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613695824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Crucible stats registered with UUID: 60e6427a-79a0-425c-9d72-2a130236b39e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.613717221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:00.613740718Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614103495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614127592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} {"msg":"60e6427a-79a0-425c-9d72-2a130236b39e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614185473Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614202521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614224498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614242265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614260562Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614285419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614310255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614334791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614358618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614383384Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"ds_connection connected from Ok(127.0.0.1:38921)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614407171Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"ds_connection connected from Ok(127.0.0.1:48503)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.614450905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"ds_connection connected from Ok(127.0.0.1:61801)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.61448444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.615130636Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.615317868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.615359802Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.615738927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.615915701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616046412Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616402701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616527472Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[0]R flush_numbers[0..12]: [219, 219, 208, 216, 219, 216, 219, 219, 206, 219, 210, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616651184Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[0]R generation[0..12]: [135, 135, 130, 132, 135, 132, 135, 135, 128, 135, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616670311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616684769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[1]R flush_numbers[0..12]: [219, 219, 208, 216, 219, 216, 219, 219, 206, 219, 210, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616698367Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[1]R generation[0..12]: [135, 135, 130, 132, 135, 132, 135, 135, 128, 135, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616711796Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616725454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[2]R flush_numbers[0..12]: [219, 219, 208, 216, 219, 216, 219, 219, 206, 219, 210, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616738821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[2]R generation[0..12]: [135, 135, 130, 132, 135, 132, 135, 135, 128, 135, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.61675249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616766497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616779246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616796933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616810111Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616828619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"":"downstairs","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616841737Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616853455Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"0","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616866563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"1","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616879741Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"client":"2","":"downstairs","session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616893069Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616905147Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"60e6427a-79a0-425c-9d72-2a130236b39e is now active with session: d5b51487-e577-4cad-bca6-6aaa045d5433","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616917506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616929224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726,"session_id":"d5b51487-e577-4cad-bca6-6aaa045d5433"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:00.616941112Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 8192 data: 13 14 02/30 Write block 568 len 36864 data: 12 12 13 15 14 15 14 13 12 03/30 Read block 369 len 4096 04/30 Read block 514 len 24576 05/30 Write block 199 len 24576 data: 11 11 10 11 7 6 06/30 Read block 366 len 4096 07/30 Write block 189 len 32768 data: 15 16 17 15 15 12 12 13 08/30 Write block 450 len 40960 data: 10 11 9 11 11 11 9 10 9 11 09/30 Read block 4 len 8192 10/30 Flush 11/30 Write block 209 len 16384 data: 14 11 10 9 12/30 Flush 13/30 Write block 363 len 36864 data: 10 10 11 12 12 11 11 10 10 14/30 Write block 27 len 24576 data: 7 9 8 10 12 15 15/30 Read block 258 len 32768 16/30 Read block 63 len 28672 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 396 len 36864 data: 13 15 16 15 13 13 13 17 18 21/30 Write block 410 len 20480 data: 15 12 14 12 13 22/30 Read block 127 len 8192 23/30 Read block 332 len 4096 24/30 Write block 370 len 24576 data: 11 11 8 6 7 9 25/30 Write block 67 len 32768 data: 10 8 12 11 10 10 10 9 26/30 Flush 27/30 Write block 422 len 8192 data: 16 13 28/30 Write block 363 len 32768 data: 11 11 12 13 13 12 12 12 29/30 Read block 353 len 32768 30/30 Write block 9 len 20480 data: 10 12 9 8 5 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:00.635627083Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 219 219 219 T T T 1 020-039 136 136 136 222 222 222 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 136 136 136 225 225 225 F F F 4 080-099 135 135 135 219 219 219 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 135 135 135 219 219 219 F F F 7 140-159 135 135 135 219 219 219 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 136 136 136 220 220 220 F F F 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 135 135 135 219 219 219 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 136 136 136 220 220 220 F F F 18 360-379 136 136 136 225 225 225 T T T 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 135 135 135 219 219 219 T T T 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 135 135 135 219 219 219 F F F 28 560-579 136 136 136 220 220 220 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 136, Max flush: 225 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 136 136 136 222 222 222 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 136 136 136 225 225 225 F F F 4 080-099 135 135 135 219 219 219 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 135 135 135 219 219 219 F F F 7 140-159 135 135 135 219 219 219 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 136 136 136 220 220 220 F F F 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 135 135 135 219 219 219 F F F 14 280-299 127 127 127 203 203 203 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 F F F 17 340-359 136 136 136 220 220 220 F F F 18 360-379 137 137 137 226 226 226 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 137 137 137 226 226 226 F F F 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 130 130 130 209 209 209 F F F 25 500-519 127 127 127 203 203 203 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 135 135 135 219 219 219 F F F 28 560-579 136 136 136 220 220 220 F F F 29 580-599 119 119 119 191 191 191 F F F Max gen: 137, Max flush: 226 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.898500013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.942383697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.942945975Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.942990728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.943006686Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.943022174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} {"msg":"Crucible 3e9d2da7-858f-4c8b-a4aa-8655dfecb1b0 has session id: 9478931e-f0be-4b16-a9c4-4b47fe044193","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.943584432Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Upstairs opts: Upstairs UUID: 3e9d2da7-858f-4c8b-a4aa-8655dfecb1b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.943626476Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Crucible stats registered with UUID: 3e9d2da7-858f-4c8b-a4aa-8655dfecb1b0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.943650062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:04.943674599Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944097107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944125303Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} {"msg":"3e9d2da7-858f-4c8b-a4aa-8655dfecb1b0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944153099Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944167417Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944189984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944204102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.94421887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944232268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944246246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944259574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944273222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944293669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:04.944337122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:04.944369768Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"ds_connection connected from Ok(127.0.0.1:45821)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944401213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:04.944429829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"ds_connection connected from Ok(127.0.0.1:51093)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944453505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.944667744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.945123848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.945161082Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.945767094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.94579595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.946318454Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:04.94634815Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.945757605Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"ds_connection connected from Ok(127.0.0.1:39426)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.945844053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.946969839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.947586939Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[0]R flush_numbers[0..12]: [226, 222, 208, 225, 219, 216, 219, 219, 206, 220, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948155126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[0]R generation[0..12]: [137, 136, 130, 136, 135, 132, 135, 135, 128, 136, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948185062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948201369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[1]R flush_numbers[0..12]: [226, 222, 208, 225, 219, 216, 219, 219, 206, 220, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948216037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[1]R generation[0..12]: [137, 136, 130, 136, 135, 132, 135, 135, 128, 136, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948230025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948244013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[2]R flush_numbers[0..12]: [226, 222, 208, 225, 219, 216, 219, 219, 206, 220, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948257641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[2]R generation[0..12]: [137, 136, 130, 136, 135, 132, 135, 135, 128, 136, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948271269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948285287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948298265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948311303Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948324092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.94833672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"":"downstairs","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948349918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948361766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"0","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948377164Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"1","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948390742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"client":"2","":"downstairs","session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.9484041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948416348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} Wait for a query_work_queue command to finish before sending IO {"msg":"3e9d2da7-858f-4c8b-a4aa-8655dfecb1b0 is now active with session: 9478931e-f0be-4b16-a9c4-4b47fe044193","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948436005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948447943Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734,"session_id":"9478931e-f0be-4b16-a9c4-4b47fe044193"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:14.948459962Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 279 len 32768 data: 10 11 13 12 12 15 17 14 03/30 Read block 328 len 24576 04/30 Read block 258 len 36864 05/30 Read block 263 len 8192 06/30 Flush 07/30 Write block 269 len 16384 data: 10 9 9 13 08/30 Flush 09/30 Write block 494 len 28672 data: 4 5 7 9 8 13 13 10/30 Flush 11/30 Write block 574 len 32768 data: 15 14 13 13 10 8 7 10 12/30 Flush 13/30 Write block 557 len 16384 data: 11 12 12 9 14/30 Read block 163 len 8192 15/30 Read block 125 len 24576 16/30 Write block 144 len 8192 data: 14 16 17/30 Read block 348 len 28672 18/30 Read block 229 len 24576 19/30 Read block 403 len 16384 20/30 Read block 554 len 24576 21/30 Read block 544 len 32768 22/30 Read block 212 len 8192 23/30 Read block 453 len 28672 24/30 Read block 544 len 40960 25/30 Flush 26/30 Write block 332 len 40960 data: 16 13 13 12 11 12 14 15 12 9 27/30 Read block 35 len 32768 28/30 Write block 183 len 16384 data: 9 12 11 12 29/30 Write block 329 len 24576 data: 21 18 17 17 14 14 30/30 Write block 26 len 40960 data: 8 8 10 9 11 13 16 13 13 11 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:14.969313337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 136 136 136 222 222 222 T T T 2 040-059 130 130 130 208 208 208 F F F 3 060-079 136 136 136 225 225 225 F F F 4 080-099 135 135 135 219 219 219 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 135 135 135 219 219 219 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 136 136 136 220 220 220 T T T 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 138 138 138 229 229 229 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 131 131 131 210 210 210 T T T 17 340-359 136 136 136 220 220 220 T T T 18 360-379 137 137 137 226 226 226 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 137 137 137 226 226 226 F F F 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 138 138 138 230 230 230 F F F 25 500-519 138 138 138 230 230 230 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 138 138 138 232 232 232 F F F 29 580-599 138 138 138 231 231 231 F F F Max gen: 138, Max flush: 232 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 139 139 139 233 233 233 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 136 136 136 225 225 225 F F F 4 080-099 135 135 135 219 219 219 F F F 5 100-119 132 132 132 216 216 216 F F F 6 120-139 135 135 135 219 219 219 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 128 128 128 206 206 206 F F F 9 180-199 139 139 139 233 233 233 F F F 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 F F F 13 260-279 138 138 138 229 229 229 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 139 139 139 233 233 233 F F F 17 340-359 139 139 139 233 233 233 F F F 18 360-379 137 137 137 226 226 226 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 137 137 137 226 226 226 F F F 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 138 138 138 230 230 230 F F F 25 500-519 138 138 138 230 230 230 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 138 138 138 232 232 232 F F F 29 580-599 138 138 138 231 231 231 F F F Max gen: 139, Max flush: 233 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.206176234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.251125152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.251748932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.251793285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.251808003Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.251822291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} {"msg":"Crucible be067881-1c28-453a-aaf2-43eb168a7682 has session id: 52d5da51-dd8a-483a-9842-834a22cf9858","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252348834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Upstairs opts: Upstairs UUID: be067881-1c28-453a-aaf2-43eb168a7682, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252390988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Crucible stats registered with UUID: be067881-1c28-453a-aaf2-43eb168a7682","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252407126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:19.252420334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252848301Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252867809Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} {"msg":"be067881-1c28-453a-aaf2-43eb168a7682 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252922011Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252947907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.252974783Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253033914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253067889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253089206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253115063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253142758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253170205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25320011Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"ds_connection connected from Ok(127.0.0.1:60128)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253230896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"ds_connection connected from Ok(127.0.0.1:58345)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253261191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"ds_connection connected from Ok(127.0.0.1:50921)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25340999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253869693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.253898468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.254053976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25457746Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.254612285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.254638431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255164824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255201689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[0]R flush_numbers[0..12]: [226, 233, 208, 225, 219, 216, 219, 232, 206, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255244453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[0]R generation[0..12]: [137, 139, 130, 136, 135, 132, 135, 138, 128, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25526261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255277108Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[1]R flush_numbers[0..12]: [226, 233, 208, 225, 219, 216, 219, 232, 206, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255290776Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[1]R generation[0..12]: [137, 139, 130, 136, 135, 132, 135, 138, 128, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255304234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255317782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[2]R flush_numbers[0..12]: [226, 233, 208, 225, 219, 216, 219, 232, 206, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25533135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[2]R generation[0..12]: [137, 139, 130, 136, 135, 132, 135, 138, 128, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255344868Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255358376Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255370994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255388332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25540181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255414778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"":"downstairs","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255427716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"Wait for a query_work_queue command to finish before sending IO 52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255449503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"0","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255462691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"1","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255475599Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"client":"2","":"downstairs","session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255488627Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255500546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"be067881-1c28-453a-aaf2-43eb168a7682 is now active with session: 52d5da51-dd8a-483a-9842-834a22cf9858","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255513053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.255524812Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742,"session_id":"52d5da51-dd8a-483a-9842-834a22cf9858"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:19.25553666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 343 len 40960 data: 12 14 15 15 15 14 15 14 14 12 02/30 Write block 128 len 8192 data: 5 8 03/30 Write block 154 len 36864 data: 10 10 11 10 8 7 8 10 8 04/30 Read block 56 len 36864 05/30 Read block 402 len 24576 06/30 Read block 268 len 32768 07/30 Read block 61 len 16384 08/30 Write block 139 len 40960 data: 8 7 10 12 12 15 17 13 14 17 09/30 Read block 497 len 12288 10/30 Write block 568 len 4096 data: 13 11/30 Flush 12/30 Write block 31 len 12288 data: 14 17 14 13/30 Read block 77 len 20480 14/30 Write block 572 len 20480 data: 15 16 16 15 14 15/30 Write block 346 len 4096 data: 16 16/30 Read block 371 len 20480 17/30 Read block 421 len 40960 18/30 Write block 88 len 28672 data: 7 8 7 7 7 6 6 19/30 Write block 368 len 32768 data: 13 13 13 12 9 7 8 10 20/30 Write block 259 len 28672 data: 8 7 7 7 8 11 13 21/30 Read block 325 len 24576 22/30 Read block 358 len 16384 23/30 Write block 65 len 36864 data: 18 14 11 9 13 12 11 11 11 24/30 Read block 424 len 24576 25/30 Write block 517 len 4096 data: 9 26/30 Read block 556 len 40960 27/30 Read block 500 len 16384 28/30 Read block 467 len 4096 29/30 Write block 100 len 24576 data: 15 15 12 11 14 11 30/30 Read block 182 len 4096 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:19.274650158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 139 139 139 233 233 233 T T T 2 040-059 130 130 130 208 208 208 F F F 3 060-079 136 136 136 225 225 225 T T T 4 080-099 135 135 135 219 219 219 T T T 5 100-119 132 132 132 216 216 216 T T T 6 120-139 140 140 140 234 234 234 F F F 7 140-159 140 140 140 234 234 234 F F F 8 160-179 140 140 140 234 234 234 F F F 9 180-199 139 139 139 233 233 233 F F F 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 130 130 130 209 209 209 T T T 13 260-279 138 138 138 229 229 229 T T T 14 280-299 138 138 138 228 228 228 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 139 139 139 233 233 233 F F F 17 340-359 140 140 140 234 234 234 T T T 18 360-379 137 137 137 226 226 226 T T T 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 137 137 137 226 226 226 F F F 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 138 138 138 230 230 230 F F F 25 500-519 138 138 138 230 230 230 T T T 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 140 140 140 234 234 234 T T T 29 580-599 138 138 138 231 231 231 F F F Max gen: 140, Max flush: 234 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 141 141 141 235 235 235 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 141 141 141 235 235 235 F F F 6 120-139 140 140 140 234 234 234 F F F 7 140-159 140 140 140 234 234 234 F F F 8 160-179 140 140 140 234 234 234 F F F 9 180-199 139 139 139 233 233 233 F F F 10 200-219 136 136 136 221 221 221 F F F 11 220-239 135 135 135 219 219 219 F F F 12 240-259 141 141 141 235 235 235 F F F 13 260-279 141 141 141 235 235 235 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 133 133 133 217 217 217 F F F 16 320-339 139 139 139 233 233 233 F F F 17 340-359 141 141 141 235 235 235 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 137 137 137 226 226 226 F F F 22 440-459 136 136 136 220 220 220 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 138 138 138 230 230 230 F F F 25 500-519 141 141 141 235 235 235 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 141 141 141 235 235 235 F F F 29 580-599 138 138 138 231 231 231 F F F Max gen: 141, Max flush: 235 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.537185089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:23.580435424Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.580823138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.61262168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613218643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613257647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613274305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613291662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} {"msg":"Crucible ef11b1e8-bd16-4c2b-beb1-541b8ed55ed4 has session id: 5e114c1e-3338-4e11-9cfd-b8f19000e018","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613823875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Upstairs opts: Upstairs UUID: ef11b1e8-bd16-4c2b-beb1-541b8ed55ed4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613865389Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Crucible stats registered with UUID: ef11b1e8-bd16-4c2b-beb1-541b8ed55ed4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.613884346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:23.613899994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614436596Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614463222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} {"msg":"ef11b1e8-bd16-4c2b-beb1-541b8ed55ed4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614535481Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614552369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614574206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614588304Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.61461273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614638206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614666612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614689739Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614716865Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614745331Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"ds_connection connected from Ok(127.0.0.1:45786)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614765138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"ds_connection connected from Ok(127.0.0.1:55938)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614802213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"ds_connection connected from Ok(127.0.0.1:41154)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.614834218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.615613855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.615651689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.615717209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.616311063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.616450823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.616479138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.616862383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.61701965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[0]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 234, 234, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617129124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[0]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 140, 140, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617150041Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617168508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[1]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 234, 234, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617182546Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[1]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 140, 140, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617196004Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617209662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[2]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 234, 234, 233, 221, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.61722306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[2]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 140, 140, 139, 136, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617236348Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617249817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617262415Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617275063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617287581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617300259Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"":"downstairs","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617320616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617332214Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"0","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617345423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"1","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.61736127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"client":"2","":"downstairs","session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617374698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617386766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"ef11b1e8-bd16-4c2b-beb1-541b8ed55ed4 is now active with session: 5e114c1e-3338-4e11-9cfd-b8f19000e018","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617399475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617411053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750,"session_id":"5e114c1e-3338-4e11-9cfd-b8f19000e018"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:23.617422821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 332 len 4096 data: 18 02/30 Read block 113 len 4096 03/30 Read block 527 len 36864 04/30 Read block 343 len 32768 05/30 Flush 06/30 Write block 488 len 20480 data: 8 10 8 7 6 07/30 Write block 425 len 36864 data: 13 9 8 11 12 14 14 12 11 08/30 Write block 441 len 32768 data: 10 11 14 14 12 10 13 11 09/30 Write block 221 len 40960 data: 9 10 10 9 8 11 10 7 9 10 10/30 Read block 227 len 12288 11/30 Write block 207 len 32768 data: 8 11 15 12 11 10 7 8 12/30 Read block 350 len 20480 13/30 Read block 269 len 32768 14/30 Read block 498 len 20480 15/30 Read block 262 len 4096 16/30 Write block 451 len 32768 data: 12 10 12 12 12 10 11 10 17/30 Read block 220 len 24576 18/30 Write block 158 len 36864 data: 9 8 9 11 9 7 8 10 10 19/30 Read block 290 len 12288 20/30 Flush 21/30 Write block 560 len 40960 data: 10 8 9 9 7 6 8 10 14 13 22/30 Read block 559 len 12288 23/30 Flush 24/30 Write block 307 len 36864 data: 8 8 9 8 8 12 12 10 10 25/30 Write block 596 len 8192 data: 7 6 26/30 Read block 264 len 12288 27/30 Write block 323 len 36864 data: 10 13 14 14 16 16 22 19 18 28/30 Read block 355 len 8192 29/30 Write block 198 len 16384 data: 14 12 12 11 30/30 Write block 248 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:23.637473153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 141 141 141 235 235 235 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 141 141 141 235 235 235 F F F 6 120-139 140 140 140 234 234 234 F F F 7 140-159 142 142 142 237 237 237 F F F 8 160-179 142 142 142 237 237 237 F F F 9 180-199 139 139 139 233 233 233 T T T 10 200-219 142 142 142 237 237 237 T T T 11 220-239 142 142 142 237 237 237 F F F 12 240-259 141 141 141 235 235 235 T T T 13 260-279 141 141 141 235 235 235 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 133 133 133 217 217 217 T T T 16 320-339 142 142 142 236 236 236 T T T 17 340-359 141 141 141 235 235 235 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 142 142 142 237 237 237 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 141 141 141 235 235 235 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 142 142 142 238 238 238 F F F 29 580-599 138 138 138 231 231 231 T T T Max gen: 142, Max flush: 238 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 141 141 141 235 235 235 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 141 141 141 235 235 235 F F F 6 120-139 140 140 140 234 234 234 F F F 7 140-159 142 142 142 237 237 237 F F F 8 160-179 142 142 142 237 237 237 F F F 9 180-199 143 143 143 239 239 239 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 F F F 12 240-259 143 143 143 239 239 239 F F F 13 260-279 141 141 141 235 235 235 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 143 143 143 239 239 239 F F F 16 320-339 143 143 143 239 239 239 F F F 17 340-359 141 141 141 235 235 235 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 136 136 136 225 225 225 F F F 20 400-419 136 136 136 225 225 225 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 142 142 142 237 237 237 F F F 23 460-479 134 134 134 218 218 218 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 141 141 141 235 235 235 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 142 142 142 238 238 238 F F F 29 580-599 143 143 143 239 239 239 F F F Max gen: 143, Max flush: 239 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.911045597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:27.953675273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.954030201Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.985379449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.98592058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.985955385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.985969543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.985984061Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} {"msg":"Crucible e531cd54-d6b6-46ad-b176-a0b9ca04f2a4 has session id: 6ec22ef5-2b57-4b53-9151-df37302f18e1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.986644155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Upstairs opts: Upstairs UUID: e531cd54-d6b6-46ad-b176-a0b9ca04f2a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.986687238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Crucible stats registered with UUID: e531cd54-d6b6-46ad-b176-a0b9ca04f2a4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.986706555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:27.986722813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987100078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987123595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} {"msg":"e531cd54-d6b6-46ad-b176-a0b9ca04f2a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987172288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987199644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987228879Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987243907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987258775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987272013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987285681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987333864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987350682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987403174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45900)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987439419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47491)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987464665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35571)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.987508089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988167083Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988195709Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988211296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988832656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988860702Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.988887298Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989402033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989504318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[0]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 237, 237, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.98956238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[0]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989581507Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989595965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[1]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 237, 237, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989609593Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[1]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989623081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989636689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[2]R flush_numbers[0..12]: [226, 235, 208, 235, 235, 235, 234, 237, 237, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989649977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[2]R generation[0..12]: [137, 141, 130, 141, 141, 141, 140, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989663735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989677333Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989693881Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989707199Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989719717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989732555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"":"downstairs","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989745733Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989757442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"0","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989778299Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"1","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989791326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"client":"2","":"downstairs","session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989804265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989816283Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"e531cd54-d6b6-46ad-b176-a0b9ca04f2a4 is now active with session: 6ec22ef5-2b57-4b53-9151-df37302f18e1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989828701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989840299Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758,"session_id":"6ec22ef5-2b57-4b53-9151-df37302f18e1"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:27.989852228Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 312 len 24576 data: 13 13 11 11 9 10 02/30 Write block 505 len 12288 data: 16 16 20 03/30 Write block 77 len 20480 data: 11 9 9 9 6 04/30 Write block 130 len 4096 data: 9 05/30 Read block 540 len 8192 06/30 Write block 399 len 36864 data: 16 14 14 14 18 19 18 16 15 07/30 Write block 160 len 24576 data: 10 12 10 8 9 11 08/30 Write block 459 len 36864 data: 12 13 10 13 11 11 12 12 11 09/30 Read block 302 len 4096 10/30 Write block 70 len 4096 data: 13 11/30 Read block 151 len 36864 12/30 Read block 415 len 20480 13/30 Read block 309 len 24576 14/30 Read block 358 len 8192 15/30 Read block 263 len 28672 16/30 Flush 17/30 Write block 148 len 40960 data: 18 20 15 14 12 11 11 11 12 11 18/30 Read block 115 len 24576 19/30 Write block 332 len 40960 data: 19 15 15 13 12 13 15 16 13 10 20/30 Write block 140 len 40960 data: 8 11 13 13 16 18 14 15 19 21 21/30 Write block 391 len 8192 data: 15 16 22/30 Read block 339 len 36864 23/30 Write block 452 len 32768 data: 11 13 13 13 11 12 11 13 24/30 Read block 386 len 16384 25/30 Read block 404 len 16384 26/30 Write block 551 len 28672 data: 10 8 9 7 11 13 12 27/30 Read block 166 len 20480 28/30 Read block 525 len 4096 29/30 Write block 263 len 36864 data: 9 12 14 14 12 10 11 10 10 30/30 Read block 537 len 20480 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:28.010136996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 144 144 144 240 240 240 F F F 4 080-099 144 144 144 240 240 240 F F F 5 100-119 141 141 141 235 235 235 F F F 6 120-139 144 144 144 240 240 240 F F F 7 140-159 142 142 142 237 237 237 T T T 8 160-179 144 144 144 240 240 240 F F F 9 180-199 143 143 143 239 239 239 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 F F F 12 240-259 143 143 143 239 239 239 F F F 13 260-279 141 141 141 235 235 235 T T T 14 280-299 138 138 138 228 228 228 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 143 143 143 239 239 239 T T T 17 340-359 141 141 141 235 235 235 T T T 18 360-379 141 141 141 235 235 235 F F F 19 380-399 144 144 144 240 240 240 T T T 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 144 144 144 240 240 240 T T T 23 460-479 144 144 144 240 240 240 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 144 144 144 240 240 240 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 138 138 138 232 232 232 T T T 28 560-579 142 142 142 238 238 238 F F F 29 580-599 143 143 143 239 239 239 F F F Max gen: 144, Max flush: 240 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 F F F 3 060-079 144 144 144 240 240 240 F F F 4 080-099 144 144 144 240 240 240 F F F 5 100-119 141 141 141 235 235 235 F F F 6 120-139 144 144 144 240 240 240 F F F 7 140-159 145 145 145 241 241 241 F F F 8 160-179 144 144 144 240 240 240 F F F 9 180-199 143 143 143 239 239 239 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 F F F 12 240-259 143 143 143 239 239 239 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 138 138 138 228 228 228 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 145 145 145 241 241 241 F F F 17 340-359 145 145 145 241 241 241 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 145 145 145 241 241 241 F F F 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 145 145 145 241 241 241 F F F 23 460-479 144 144 144 240 240 240 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 144 144 144 240 240 240 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 145 145 145 241 241 241 F F F 28 560-579 142 142 142 238 238 238 F F F 29 580-599 143 143 143 239 239 239 F F F Max gen: 145, Max flush: 241 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.28611222Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.330830221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.331585821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.331641233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.331660141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.331678898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} {"msg":"Crucible 32cba6de-2e14-4b8a-a917-77bfd3cccad1 has session id: f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.332369137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Upstairs opts: Upstairs UUID: 32cba6de-2e14-4b8a-a917-77bfd3cccad1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.332405272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Crucible stats registered with UUID: 32cba6de-2e14-4b8a-a917-77bfd3cccad1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33242137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:32.332434468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33290306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.332927416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} {"msg":"32cba6de-2e14-4b8a-a917-77bfd3cccad1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.332985678Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33303744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333064876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333090442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333116489Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333204936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333235401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333260078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333283644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33331245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:57889)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333506042Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61932)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333545266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63255)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.333610767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.334007179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.334189482Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.334695899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.334722945Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.334743402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335348744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335488973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335507671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[0]R flush_numbers[0..12]: [226, 235, 208, 240, 240, 235, 240, 241, 240, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335916971Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[0]R generation[0..12]: [137, 141, 130, 144, 144, 141, 144, 145, 144, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335936068Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335950716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[1]R flush_numbers[0..12]: [226, 235, 208, 240, 240, 235, 240, 241, 240, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335968334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[1]R generation[0..12]: [137, 141, 130, 144, 144, 141, 144, 145, 144, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.335982361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336009178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[2]R flush_numbers[0..12]: [226, 235, 208, 240, 240, 235, 240, 241, 240, 239, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336024315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[2]R generation[0..12]: [137, 141, 130, 144, 144, 141, 144, 145, 144, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336037853Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336051431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33606415Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336077198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336089896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336102504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"":"downstairs","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336115172Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336135679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"0","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336148707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"1","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336161395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"client":"2","":"downstairs","session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336176453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336188642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"32cba6de-2e14-4b8a-a917-77bfd3cccad1 is now active with session: f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.33620121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336212828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766,"session_id":"f8b8f9ad-7edd-47bd-ae08-b18dd74ea4b4"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:32.336224576Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 8192 data: 14 14 02/30 Write block 289 len 28672 data: 14 13 11 10 11 13 15 03/30 Read block 314 len 20480 04/30 Read block 13 len 36864 05/30 Read block 507 len 36864 06/30 Read block 192 len 4096 07/30 Read block 91 len 32768 08/30 Write block 455 len 28672 data: 14 12 13 12 14 14 11 09/30 Write block 458 len 36864 data: 13 15 15 12 14 12 12 13 13 10/30 Write block 118 len 4096 data: 4 11/30 Read block 412 len 36864 12/30 Write block 85 len 12288 data: 4 5 7 13/30 Read block 579 len 28672 14/30 Read block 173 len 28672 15/30 Read block 332 len 12288 16/30 Flush 17/30 Write block 197 len 4096 data: 13 18/30 Read block 320 len 40960 19/30 Read block 297 len 28672 20/30 Read block 139 len 8192 21/30 Write block 553 len 36864 data: 10 8 12 14 13 13 13 11 9 22/30 Write block 53 len 32768 data: 10 9 8 8 10 9 6 6 23/30 Read block 428 len 28672 24/30 Write block 79 len 12288 data: 10 10 7 25/30 Read block 311 len 4096 26/30 Write block 247 len 32768 data: 9 9 9 7 5 6 6 7 27/30 Write block 243 len 20480 data: 14 11 11 10 10 28/30 Read block 276 len 28672 29/30 Read block 164 len 24576 30/30 Read block 167 len 36864 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 14 13 14 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:32.356151386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 130 130 130 208 208 208 T T T 3 060-079 144 144 144 240 240 240 T T T 4 080-099 146 146 146 242 242 242 T T T 5 100-119 146 146 146 242 242 242 F F F 6 120-139 144 144 144 240 240 240 F F F 7 140-159 145 145 145 241 241 241 F F F 8 160-179 144 144 144 240 240 240 F F F 9 180-199 143 143 143 239 239 239 T T T 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 F F F 12 240-259 143 143 143 239 239 239 T T T 13 260-279 145 145 145 241 241 241 F F F 14 280-299 146 146 146 242 242 242 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 145 145 145 241 241 241 F F F 17 340-359 145 145 145 241 241 241 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 145 145 145 241 241 241 F F F 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 146 146 146 242 242 242 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 144 144 144 240 240 240 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 145 145 145 241 241 241 T T T 28 560-579 142 142 142 238 238 238 T T T 29 580-599 143 143 143 239 239 239 F F F Max gen: 146, Max flush: 242 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 147 147 147 243 243 243 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 146 146 146 242 242 242 F F F 6 120-139 144 144 144 240 240 240 F F F 7 140-159 145 145 145 241 241 241 F F F 8 160-179 144 144 144 240 240 240 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 F F F 12 240-259 147 147 147 243 243 243 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 146 146 146 242 242 242 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 145 145 145 241 241 241 F F F 17 340-359 145 145 145 241 241 241 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 145 145 145 241 241 241 F F F 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 146 146 146 242 242 242 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 144 144 144 240 240 240 F F F 26 520-539 135 135 135 219 219 219 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 147 147 147 243 243 243 F F F 29 580-599 143 143 143 239 239 239 F F F Max gen: 147, Max flush: 243 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.607620578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:36.650363987Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.650728164Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.682303848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.682847019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.682889163Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.68290661Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.682924848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} {"msg":"Crucible f07dcdac-fd1f-48d7-b97a-72eaf8ca0f4b has session id: a99777f8-d03a-4126-b4b6-40752b2b0e11","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.683615617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Upstairs opts: Upstairs UUID: f07dcdac-fd1f-48d7-b97a-72eaf8ca0f4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.683670689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Crucible stats registered with UUID: f07dcdac-fd1f-48d7-b97a-72eaf8ca0f4b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.683702475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:36.683724922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684126453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684152379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} {"msg":"f07dcdac-fd1f-48d7-b97a-72eaf8ca0f4b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.68421423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684236887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684260753Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684274691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684288769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684301478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684315176Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684327814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684341202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.68435443Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:54351)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684410612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:42359)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684512227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:54013)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.684549262Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.685063727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.68510533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.685267037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.685706673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.685748707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.685852592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686245385Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686292738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[0]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 242, 240, 241, 240, 243, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686456444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[0]R generation[0..12]: [137, 141, 147, 147, 147, 146, 144, 145, 144, 147, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686476111Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686490899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[1]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 242, 240, 241, 240, 243, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686504997Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[1]R generation[0..12]: [137, 141, 147, 147, 147, 146, 144, 145, 144, 147, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686518755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686532463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[2]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 242, 240, 241, 240, 243, 239, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686546011Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[2]R generation[0..12]: [137, 141, 147, 147, 147, 146, 144, 145, 144, 147, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686559399Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686577106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686590114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686603012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686615471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:16:36.686628079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"":"downstairs","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686648806Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686660414Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"0","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686673502Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"1","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.68668626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"client":"2","":"downstairs","session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686700038Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686712027Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"f07dcdac-fd1f-48d7-b97a-72eaf8ca0f4b is now active with session: a99777f8-d03a-4126-b4b6-40752b2b0e11","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686724485Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686736103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774,"session_id":"a99777f8-d03a-4126-b4b6-40752b2b0e11"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:36.686748002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 586 len 20480 data: 12 9 10 9 8 03/30 Read block 138 len 8192 04/30 Read block 14 len 4096 05/30 Write block 507 len 40960 data: 21 20 16 13 11 10 9 8 8 8 06/30 Read block 433 len 12288 07/30 Write block 472 len 32768 data: 13 12 12 14 14 14 12 10 08/30 Read block 371 len 8192 09/30 Read block 77 len 24576 10/30 Write block 131 len 16384 data: 9 10 12 13 11/30 Write block 517 len 20480 data: 10 12 13 12 10 12/30 Read block 74 len 36864 13/30 Read block 393 len 8192 14/30 Write block 389 len 24576 data: 15 17 16 17 15 12 15/30 Flush 16/30 Write block 157 len 16384 data: 12 10 9 11 17/30 Write block 582 len 40960 data: 11 7 11 12 13 10 11 10 9 8 18/30 Write block 232 len 4096 data: 15 19/30 Read block 204 len 8192 20/30 Write block 573 len 16384 data: 17 17 16 15 21/30 Read block 320 len 20480 22/30 Write block 127 len 28672 data: 4 6 9 10 10 11 13 23/30 Read block 550 len 20480 24/30 Read block 26 len 36864 25/30 Write block 587 len 8192 data: 11 12 26/30 Read block 435 len 24576 27/30 Read block 500 len 24576 28/30 Write block 115 len 28672 data: 6 5 5 5 4 6 6 29/30 Read block 443 len 28672 30/30 Write block 339 len 12288 data: 17 14 11 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 4 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:36.706333091Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 147 147 147 243 243 243 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 146 146 146 242 242 242 T T T 6 120-139 148 148 148 245 245 245 T T T 7 140-159 145 145 145 241 241 241 T T T 8 160-179 144 144 144 240 240 240 T T T 9 180-199 147 147 147 243 243 243 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 142 142 142 237 237 237 T T T 12 240-259 147 147 147 243 243 243 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 146 146 146 242 242 242 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 145 145 145 241 241 241 T T T 17 340-359 145 145 145 241 241 241 T T T 18 360-379 141 141 141 235 235 235 F F F 19 380-399 148 148 148 245 245 245 F F F 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 148 148 148 245 245 245 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 147 147 147 243 243 243 T T T 29 580-599 148 148 148 245 245 245 T T T Max gen: 148, Max flush: 245 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 226 226 226 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 147 147 147 243 243 243 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 149 149 149 246 246 246 F F F 6 120-139 149 149 149 246 246 246 F F F 7 140-159 149 149 149 246 246 246 F F F 8 160-179 149 149 149 246 246 246 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 143 143 143 239 239 239 F F F 11 220-239 149 149 149 246 246 246 F F F 12 240-259 147 147 147 243 243 243 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 146 146 146 242 242 242 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 149 149 149 246 246 246 F F F 17 340-359 149 149 149 246 246 246 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 148 148 148 245 245 245 F F F 20 400-419 144 144 144 240 240 240 F F F 21 420-439 142 142 142 237 237 237 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 148 148 148 245 245 245 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 F F F Max gen: 149, Max flush: 246 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:40.994649139Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.040122491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.040709595Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.040751929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.040767287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.040782695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} {"msg":"Crucible 4c1a5101-42e8-4ee8-af49-ea389e97d925 has session id: 26766973-3194-4d92-8dc2-470e1610a253","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.041511699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Upstairs opts: Upstairs UUID: 4c1a5101-42e8-4ee8-af49-ea389e97d925, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.041557212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Crucible stats registered with UUID: 4c1a5101-42e8-4ee8-af49-ea389e97d925","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.04157538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:41.041588967Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.041968362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.041991529Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} {"msg":"4c1a5101-42e8-4ee8-af49-ea389e97d925 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.0420523Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042078646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042102493Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042117921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042132598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042145697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042159495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042172282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042185611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042201878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"ds_connection connected from Ok(127.0.0.1:54423)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042288276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"ds_connection connected from Ok(127.0.0.1:64034)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.04232858Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"ds_connection connected from Ok(127.0.0.1:49278)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.042444743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.043037887Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.043066283Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.04308245Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.043599485Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.043746833Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.043774769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044131197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044306792Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[0]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 246, 246, 246, 246, 243, 239, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044414376Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[0]R generation[0..12]: [137, 141, 147, 147, 147, 149, 149, 149, 149, 147, 143, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044433453Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044447911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[1]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 246, 246, 246, 246, 243, 239, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044465969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[1]R generation[0..12]: [137, 141, 147, 147, 147, 149, 149, 149, 149, 147, 143, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044480087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044493885Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[2]R flush_numbers[0..12]: [226, 235, 243, 243, 243, 246, 246, 246, 246, 243, 239, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044509702Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[2]R generation[0..12]: [137, 141, 147, 147, 147, 149, 149, 149, 149, 147, 143, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044533999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044561835Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044582232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044601039Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044629275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044654181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"":"downstairs","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044676748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044689606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"0","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044702574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"1","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044721022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"client":"2","":"downstairs","session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044747328Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044769404Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"4c1a5101-42e8-4ee8-af49-ea389e97d925 is now active with session: 26766973-3194-4d92-8dc2-470e1610a253","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044791801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044813628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782,"session_id":"26766973-3194-4d92-8dc2-470e1610a253"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:41.044836835Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 399 len 28672 data: 17 15 15 15 19 20 19 02/30 Write block 249 len 28672 data: 10 8 6 7 7 8 7 03/30 Write block 215 len 20480 data: 7 9 7 6 6 04/30 Read block 182 len 32768 05/30 Write block 61 len 20480 data: 9 13 17 16 19 06/30 Write block 15 len 4096 data: 8 07/30 Write block 55 len 16384 data: 9 9 11 10 08/30 Read block 177 len 40960 09/30 Write block 334 len 8192 data: 16 14 10/30 Read block 228 len 4096 11/30 Flush 12/30 Flush 13/30 Write block 288 len 32768 data: 13 15 14 12 11 12 14 16 14/30 Flush 15/30 Write block 233 len 40960 data: 12 14 16 16 13 15 16 17 15 17 16/30 Write block 550 len 4096 data: 9 17/30 Read block 347 len 28672 18/30 Write block 385 len 16384 data: 10 10 10 14 19/30 Read block 488 len 20480 20/30 Write block 217 len 36864 data: 8 7 7 8 10 11 11 10 9 21/30 Flush 22/30 Write block 471 len 4096 data: 11 23/30 Read block 266 len 28672 24/30 Write block 173 len 40960 data: 11 9 8 9 9 9 9 9 6 6 25/30 Read block 371 len 8192 26/30 Write block 85 len 16384 data: 5 6 8 8 27/30 Write block 198 len 20480 data: 15 13 13 12 12 28/30 Read block 581 len 32768 29/30 Write block 543 len 16384 data: 12 10 11 10 30/30 Write block 431 len 8192 data: 15 13 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:41.062968026Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 147 147 147 243 243 243 T T T 5 100-119 149 149 149 246 246 246 F F F 6 120-139 149 149 149 246 246 246 F F F 7 140-159 149 149 149 246 246 246 F F F 8 160-179 149 149 149 246 246 246 T T T 9 180-199 147 147 147 243 243 243 T T T 10 200-219 150 150 150 250 250 250 T T T 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 150 150 150 247 247 247 F F F 17 340-359 149 149 149 246 246 246 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 150 150 150 250 250 250 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 142 142 142 237 237 237 T T T 22 440-459 146 146 146 242 242 242 F F F 23 460-479 148 148 148 245 245 245 T T T 24 480-499 142 142 142 237 237 237 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 150 150 150 250 250 250 T T T 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 F F F Max gen: 150, Max flush: 250 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 141 141 141 235 235 235 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 151 151 151 251 251 251 F F F 5 100-119 149 149 149 246 246 246 F F F 6 120-139 149 149 149 246 246 246 F F F 7 140-159 149 149 149 246 246 246 F F F 8 160-179 151 151 151 251 251 251 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 151 151 151 251 251 251 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 F F F 13 260-279 145 145 145 241 241 241 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 144 144 144 240 240 240 F F F 16 320-339 150 150 150 247 247 247 F F F 17 340-359 149 149 149 246 246 246 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 150 150 150 250 250 250 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 151 151 151 251 251 251 F F F 24 480-499 142 142 142 237 237 237 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 151 151 151 251 251 251 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 F F F Max gen: 151, Max flush: 251 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.313363534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:45.358084965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.3584669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.391536267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392101325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392143308Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392160506Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392178663Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} {"msg":"Crucible a23d1e67-64f2-4abc-b9a1-2a8b089692ee has session id: 7d65c05c-ce5c-4d84-876b-d6352e0b586d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392781466Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Upstairs opts: Upstairs UUID: a23d1e67-64f2-4abc-b9a1-2a8b089692ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39281782Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Crucible stats registered with UUID: a23d1e67-64f2-4abc-b9a1-2a8b089692ee","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.392834318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:45.392847846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393281503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393314738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} {"msg":"a23d1e67-64f2-4abc-b9a1-2a8b089692ee active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39337025Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393398056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393425812Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39344155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393456637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393470995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393485303Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393498101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393511579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393525487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59143)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393546234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44729)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393649479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64733)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.393696322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.394314572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.394350007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.394493846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395042717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395072622Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395118985Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395602365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39570458Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[0]R flush_numbers[0..12]: [247, 235, 247, 247, 251, 246, 246, 246, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395757302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[0]R generation[0..12]: [150, 141, 150, 150, 151, 149, 149, 149, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39577638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395791157Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[1]R flush_numbers[0..12]: [247, 235, 247, 247, 251, 246, 246, 246, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395805155Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[1]R generation[0..12]: [150, 141, 150, 150, 151, 149, 149, 149, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395818513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395832402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[2]R flush_numbers[0..12]: [247, 235, 247, 247, 251, 246, 246, 246, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395846089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[2]R generation[0..12]: [150, 141, 150, 150, 151, 149, 149, 149, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395859268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395872956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395889013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395902341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39591491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395927478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"":"downstairs","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:16:45.395940296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395957113Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"0","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.395969982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"1","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.39598267Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"client":"2","":"downstairs","session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.396012715Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.396027253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"a23d1e67-64f2-4abc-b9a1-2a8b089692ee is now active with session: 7d65c05c-ce5c-4d84-876b-d6352e0b586d","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.396039811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:45.3960515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790,"session_id":"7d65c05c-ce5c-4d84-876b-d6352e0b586d"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-14T16:16:45.396063588Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 300 len 16384 data: 9 9 8 8 02/30 Write block 249 len 36864 data: 11 9 7 8 8 9 8 9 10 03/30 Read block 287 len 8192 04/30 Read block 130 len 4096 05/30 Read block 337 len 28672 06/30 Write block 38 len 28672 data: 13 15 16 12 10 10 7 07/30 Write block 118 len 24576 data: 6 5 7 7 5 6 08/30 Write block 253 len 8192 data: 9 10 09/30 Write block 146 len 16384 data: 15 16 20 22 10/30 Read block 203 len 24576 11/30 Write block 254 len 28672 data: 11 9 10 11 8 9 8 12/30 Write block 84 len 12288 data: 4 6 7 13/30 Read block 224 len 8192 14/30 Read block 334 len 32768 15/30 Write block 59 len 20480 data: 7 7 10 14 18 16/30 Write block 291 len 4096 data: 13 17/30 Read block 517 len 12288 18/30 Read block 350 len 4096 19/30 Write block 487 len 32768 data: 8 9 11 9 8 7 4 5 20/30 Read block 519 len 12288 21/30 Write block 485 len 16384 data: 10 9 9 10 22/30 Read block 224 len 20480 23/30 Write block 297 len 12288 data: 11 9 9 24/30 Read block 81 len 36864 25/30 Read block 201 len 16384 26/30 Read block 99 len 32768 27/30 Write block 329 len 20480 data: 23 20 19 20 16 28/30 Read block 598 len 8192 29/30 Read block 565 len 12288 30/30 Read block 202 len 16384 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 30 29 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:45.414355496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 141 141 141 235 235 235 T T T 2 040-059 150 150 150 247 247 247 T T T 3 060-079 150 150 150 247 247 247 T T T 4 080-099 151 151 151 251 251 251 T T T 5 100-119 149 149 149 246 246 246 T T T 6 120-139 149 149 149 246 246 246 T T T 7 140-159 149 149 149 246 246 246 T T T 8 160-179 151 151 151 251 251 251 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 151 151 151 251 251 251 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 T T T 13 260-279 145 145 145 241 241 241 T T T 14 280-299 150 150 150 249 249 249 T T T 15 300-319 144 144 144 240 240 240 T T T 16 320-339 150 150 150 247 247 247 T T T 17 340-359 149 149 149 246 246 246 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 150 150 150 250 250 250 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 151 151 151 251 251 251 F F F 24 480-499 142 142 142 237 237 237 T T T 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 151 151 151 251 251 251 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 F F F Max gen: 151, Max flush: 251 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 152 152 152 252 252 252 F F F 2 040-059 152 152 152 252 252 252 F F F 3 060-079 152 152 152 252 252 252 F F F 4 080-099 152 152 152 252 252 252 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 151 151 151 251 251 251 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 151 151 151 251 251 251 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 152 152 152 252 252 252 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 152 152 152 252 252 252 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 149 149 149 246 246 246 F F F 18 360-379 141 141 141 235 235 235 F F F 19 380-399 150 150 150 250 250 250 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 146 146 146 242 242 242 F F F 23 460-479 151 151 151 251 251 251 F F F 24 480-499 152 152 152 252 252 252 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 151 151 151 251 251 251 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 F F F Max gen: 152, Max flush: 252 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.730748978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.775033233Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.775617288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.775681838Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.775714074Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77573986Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} {"msg":"Crucible c2f93016-a3d2-46d9-962e-ed94a7ceec75 has session id: 13118252-6164-4f33-8ff6-d8169110c695","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77629159Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Upstairs opts: Upstairs UUID: c2f93016-a3d2-46d9-962e-ed94a7ceec75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.776335433Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Crucible stats registered with UUID: c2f93016-a3d2-46d9-962e-ed94a7ceec75","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77635851Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:49.776378637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.776890992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.776927067Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} {"msg":"c2f93016-a3d2-46d9-962e-ed94a7ceec75 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.776961472Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.776985289Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777018704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.7770434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777127138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777158073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77718241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777207916Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777230953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777253019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"ds_connection connected from Ok(127.0.0.1:42237)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777274796Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"ds_connection connected from Ok(127.0.0.1:62054)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777299283Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"ds_connection connected from Ok(127.0.0.1:43412)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777345066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.777906914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.778084309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.778480801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.778639978Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.778668313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77903272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779273135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779346085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[0]R flush_numbers[0..12]: [247, 252, 252, 252, 252, 252, 252, 252, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77985881Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[0]R generation[0..12]: [150, 152, 152, 152, 152, 152, 152, 152, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779886186Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779902444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[1]R flush_numbers[0..12]: [247, 252, 252, 252, 252, 252, 252, 252, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779916862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[1]R generation[0..12]: [150, 152, 152, 152, 152, 152, 152, 152, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.77993052Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779944378Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[2]R flush_numbers[0..12]: [247, 252, 252, 252, 252, 252, 252, 252, 251, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779957686Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[2]R generation[0..12]: [150, 152, 152, 152, 152, 152, 152, 152, 151, 151, 151, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779971174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.779984862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Max found gen is 153","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.7799976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Generation requested: 154 >= found:153","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780010468Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:16:49.780023046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780038864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"":"downstairs","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780051762Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.78006757Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"0","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780081438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"1","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780094476Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"client":"2","":"downstairs","session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780107354Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780119422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"c2f93016-a3d2-46d9-962e-ed94a7ceec75 is now active with session: 13118252-6164-4f33-8ff6-d8169110c695","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.78013177Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780143498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798,"session_id":"13118252-6164-4f33-8ff6-d8169110c695"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:49.780155377Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 12288 data: 10 11 13 02/30 Read block 168 len 32768 03/30 Write block 473 len 24576 data: 13 13 15 15 15 13 04/30 Write block 123 len 12288 data: 7 5 3 05/30 Write block 172 len 12288 data: 8 12 10 06/30 Write block 556 len 16384 data: 15 14 14 14 07/30 Read block 58 len 40960 08/30 Read block 104 len 4096 09/30 Write block 289 len 28672 data: 16 15 14 12 13 15 17 10/30 Write block 113 len 16384 data: 10 8 7 6 11/30 Read block 391 len 16384 12/30 Write block 28 len 28672 data: 11 10 12 15 18 15 14 13/30 Write block 489 len 36864 data: 12 10 9 8 5 6 6 8 10 14/30 Write block 245 len 40960 data: 12 11 11 10 12 10 8 9 10 12 15/30 Flush 16/30 Write block 596 len 8192 data: 8 7 17/30 Read block 117 len 20480 18/30 Read block 174 len 24576 19/30 Read block 94 len 20480 20/30 Read block 245 len 28672 21/30 Write block 51 len 12288 data: 11 11 11 22/30 Read block 242 len 4096 23/30 Read block 312 len 16384 24/30 Read block 159 len 32768 25/30 Write block 399 len 4096 data: 18 26/30 Write block 333 len 36864 data: 17 17 15 13 14 16 18 15 12 27/30 Write block 359 len 16384 data: 9 9 12 11 28/30 Write block 202 len 12288 data: 13 8 7 29/30 Read block 560 len 24576 30/30 Read block 404 len 8192 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 15 14 14 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:49.801025999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 152 152 152 252 252 252 T T T 3 060-079 152 152 152 252 252 252 F F F 4 080-099 152 152 152 252 252 252 F F F 5 100-119 154 154 154 253 253 253 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 154 154 154 253 253 253 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 151 151 151 251 251 251 T T T 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 152 152 152 252 252 252 T T T 17 340-359 149 149 149 246 246 246 T T T 18 360-379 141 141 141 235 235 235 T T T 19 380-399 150 150 150 250 250 250 T T T 20 400-419 150 150 150 247 247 247 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 154 154 154 253 253 253 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 154 154 154 253 253 253 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 154 154 154 253 253 253 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 149 149 149 246 246 246 T T T Max gen: 154, Max flush: 253 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 155 155 155 254 254 254 F F F 3 060-079 152 152 152 252 252 252 F F F 4 080-099 152 152 152 252 252 252 F F F 5 100-119 154 154 154 253 253 253 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 154 154 154 253 253 253 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 155 155 155 254 254 254 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 155 155 155 254 254 254 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 155 155 155 254 254 254 F F F 19 380-399 155 155 155 254 254 254 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 154 154 154 253 253 253 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 154 154 154 253 253 253 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 154 154 154 253 253 253 F F F 28 560-579 149 149 149 246 246 246 F F F 29 580-599 155 155 155 254 254 254 F F F Max gen: 155, Max flush: 254 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.048516989Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.096067039Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.096788614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.096831538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.096846005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.096876101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} {"msg":"Crucible 0557b957-8630-46f3-a6a6-0bcc62c1d4da has session id: 0e176a84-2302-495c-8ca3-a10742fa0c5f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.097522717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Upstairs opts: Upstairs UUID: 0557b957-8630-46f3-a6a6-0bcc62c1d4da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.097558682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Crucible stats registered with UUID: 0557b957-8630-46f3-a6a6-0bcc62c1d4da","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.097573959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:54.097586548Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098099153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098125319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} {"msg":"0557b957-8630-46f3-a6a6-0bcc62c1d4da active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098220756Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098249471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098282836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098306703Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098332029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098358116Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098418547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098452972Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098482587Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098512283Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33212)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098658642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48715)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098751088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34064)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.098796862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.09949559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.099524656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.099544813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100193198Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100221494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100238332Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100775584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.10080173Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[0]R flush_numbers[0..12]: [247, 253, 254, 252, 252, 253, 253, 252, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100948528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[0]R generation[0..12]: [150, 154, 155, 152, 152, 154, 154, 152, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100968565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100982643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[1]R flush_numbers[0..12]: [247, 253, 254, 252, 252, 253, 253, 252, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.100996001Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[1]R generation[0..12]: [150, 154, 155, 152, 152, 154, 154, 152, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.10100949Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101023248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[2]R flush_numbers[0..12]: [247, 253, 254, 252, 252, 253, 253, 252, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101036646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[2]R generation[0..12]: [150, 154, 155, 152, 152, 154, 154, 152, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101054013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101067971Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101080729Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101093707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101106265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:16:54.101118794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"":"downstairs","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101135781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101147319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"0","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101160218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"1","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101172886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"client":"2","":"downstairs","session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101185704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101197472Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"0557b957-8630-46f3-a6a6-0bcc62c1d4da is now active with session: 0e176a84-2302-495c-8ca3-a10742fa0c5f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.10120987Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101221389Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806,"session_id":"0e176a84-2302-495c-8ca3-a10742fa0c5f"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:54.101233277Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 576 len 20480 data: 16 14 11 9 8 02/30 Write block 99 len 16384 data: 15 16 16 13 03/30 Write block 396 len 24576 data: 14 16 17 19 16 16 04/30 Read block 338 len 16384 05/30 Read block 240 len 16384 06/30 Read block 558 len 16384 07/30 Read block 55 len 40960 08/30 Read block 123 len 40960 09/30 Read block 207 len 16384 10/30 Read block 568 len 40960 11/30 Read block 132 len 16384 12/30 Read block 449 len 36864 13/30 Read block 166 len 8192 14/30 Read block 529 len 40960 15/30 Write block 105 len 20480 data: 12 10 15 14 12 16/30 Read block 362 len 40960 17/30 Read block 481 len 4096 18/30 Read block 293 len 20480 19/30 Read block 51 len 36864 20/30 Write block 397 len 32768 data: 17 18 20 17 17 16 20 21 21/30 Read block 9 len 36864 22/30 Read block 481 len 16384 23/30 Write block 153 len 12288 data: 12 12 12 24/30 Write block 459 len 32768 data: 16 16 13 15 13 13 14 14 25/30 Read block 388 len 40960 26/30 Read block 479 len 40960 27/30 Write block 497 len 4096 data: 11 28/30 Read block 365 len 24576 29/30 Read block 513 len 8192 30/30 Read block 555 len 12288 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 29 30 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:54.121496838Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 155 155 155 254 254 254 F F F 3 060-079 152 152 152 252 252 252 F F F 4 080-099 152 152 152 252 252 252 T T T 5 100-119 154 154 154 253 253 253 T T T 6 120-139 154 154 154 253 253 253 F F F 7 140-159 152 152 152 252 252 252 T T T 8 160-179 154 154 154 253 253 253 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 155 155 155 254 254 254 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 155 155 155 254 254 254 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 155 155 155 254 254 254 F F F 19 380-399 155 155 155 254 254 254 T T T 20 400-419 150 150 150 247 247 247 T T T 21 420-439 151 151 151 251 251 251 F F F 22 440-459 154 154 154 253 253 253 T T T 23 460-479 154 154 154 253 253 253 T T T 24 480-499 154 154 154 253 253 253 T T T 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 154 154 154 253 253 253 F F F 28 560-579 149 149 149 246 246 246 T T T 29 580-599 155 155 155 254 254 254 T T T Max gen: 155, Max flush: 254 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 155 155 155 254 254 254 F F F 3 060-079 152 152 152 252 252 252 F F F 4 080-099 156 156 156 255 255 255 F F F 5 100-119 156 156 156 255 255 255 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 156 156 156 255 255 255 F F F 8 160-179 154 154 154 253 253 253 F F F 9 180-199 151 151 151 251 251 251 F F F 10 200-219 155 155 155 254 254 254 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 155 155 155 254 254 254 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 155 155 155 254 254 254 F F F 19 380-399 156 156 156 255 255 255 F F F 20 400-419 156 156 156 255 255 255 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 156 156 156 255 255 255 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 F F F 27 540-559 154 154 154 253 253 253 F F F 28 560-579 156 156 156 255 255 255 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 156, Max flush: 255 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.385972785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:58.429284182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.429681254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46158923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462170436Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462213899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462231137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462248864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} {"msg":"Crucible bdb7d46e-88c5-4bb2-80a2-32af253a2c43 has session id: 8d2a055b-ef8d-4670-8a91-5ec40327dc10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462792235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Upstairs opts: Upstairs UUID: bdb7d46e-88c5-4bb2-80a2-32af253a2c43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462836319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Crucible stats registered with UUID: bdb7d46e-88c5-4bb2-80a2-32af253a2c43","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.462858995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:58.462872973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463352654Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.4633741Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} {"msg":"bdb7d46e-88c5-4bb2-80a2-32af253a2c43 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46344402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463461148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463485884Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463510351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463569022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463628114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463661089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463691864Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46372305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463754005Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"ds_connection connected from Ok(127.0.0.1:34846)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463880437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"ds_connection connected from Ok(127.0.0.1:52978)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463916022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"ds_connection connected from Ok(127.0.0.1:60170)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.463942748Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.464436846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46447683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.464576676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465090261Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46516727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465191876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465749275Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465775221Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[0]R flush_numbers[0..12]: [247, 253, 254, 252, 255, 255, 253, 255, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465804597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[0]R generation[0..12]: [150, 154, 155, 152, 156, 156, 154, 156, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465819944Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465834393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[1]R flush_numbers[0..12]: [247, 253, 254, 252, 255, 255, 253, 255, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46584808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[1]R generation[0..12]: [150, 154, 155, 152, 156, 156, 154, 156, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465861578Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465875376Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[2]R flush_numbers[0..12]: [247, 253, 254, 252, 255, 255, 253, 255, 253, 251, 254, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465888685Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[2]R generation[0..12]: [150, 154, 155, 152, 156, 156, 154, 156, 154, 151, 155, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465902053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465933878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465947456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465960434Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.465977342Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.46599058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"":"downstairs","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466003568Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"Wait for a query_work_queue command to finish before sending IO 8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466021685Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"0","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466034983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"1","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466047911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"client":"2","":"downstairs","session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466060849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466072928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"bdb7d46e-88c5-4bb2-80a2-32af253a2c43 is now active with session: 8d2a055b-ef8d-4670-8a91-5ec40327dc10","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466085496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466097114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814,"session_id":"8d2a055b-ef8d-4670-8a91-5ec40327dc10"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:16:58.466108942Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 124 len 8192 data: 6 4 02/30 Write block 187 len 4096 data: 12 03/30 Write block 174 len 4096 data: 11 04/30 Write block 203 len 4096 data: 9 05/30 Read block 447 len 32768 06/30 Write block 544 len 24576 data: 11 12 11 12 13 12 07/30 Write block 128 len 12288 data: 7 10 11 08/30 Read block 66 len 4096 09/30 Read block 343 len 32768 10/30 Flush 11/30 Write block 191 len 24576 data: 18 16 16 13 13 14 12/30 Read block 513 len 32768 13/30 Flush 14/30 Flush 15/30 Write block 42 len 28672 data: 11 11 8 10 12 13 11 16/30 Read block 185 len 4096 17/30 Read block 33 len 40960 18/30 Read block 427 len 4096 19/30 Read block 251 len 16384 20/30 Read block 97 len 16384 21/30 Read block 73 len 40960 22/30 Write block 63 len 40960 data: 19 17 20 15 12 10 14 14 12 12 23/30 Read block 52 len 8192 24/30 Read block 118 len 16384 25/30 Write block 536 len 8192 data: 14 16 26/30 Write block 384 len 12288 data: 8 11 11 27/30 Write block 65 len 28672 data: 21 16 13 11 15 15 13 28/30 Read block 226 len 36864 29/30 Read block 588 len 24576 30/30 Read block 449 len 32768 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 15 16 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:16:58.485623612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 155 155 155 254 254 254 T T T 3 060-079 152 152 152 252 252 252 T T T 4 080-099 156 156 156 255 255 255 F F F 5 100-119 156 156 156 255 255 255 F F F 6 120-139 158 158 158 256 256 256 F F F 7 140-159 156 156 156 255 255 255 F F F 8 160-179 158 158 158 256 256 256 F F F 9 180-199 158 158 158 257 257 257 F F F 10 200-219 158 158 158 256 256 256 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 155 155 155 254 254 254 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 155 155 155 254 254 254 F F F 19 380-399 156 156 156 255 255 255 T T T 20 400-419 156 156 156 255 255 255 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 156 156 156 255 255 255 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 148 148 148 245 245 245 T T T 27 540-559 158 158 158 256 256 256 F F F 28 560-579 156 156 156 255 255 255 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 158, Max flush: 257 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 159 159 159 258 258 258 F F F 4 080-099 156 156 156 255 255 255 F F F 5 100-119 156 156 156 255 255 255 F F F 6 120-139 158 158 158 256 256 256 F F F 7 140-159 156 156 156 255 255 255 F F F 8 160-179 158 158 158 256 256 256 F F F 9 180-199 158 158 158 257 257 257 F F F 10 200-219 158 158 158 256 256 256 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 155 155 155 254 254 254 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 155 155 155 254 254 254 F F F 19 380-399 159 159 159 258 258 258 F F F 20 400-419 156 156 156 255 255 255 F F F 21 420-439 151 151 151 251 251 251 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 156 156 156 255 255 255 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 148 148 148 245 245 245 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 158 158 158 256 256 256 F F F 28 560-579 156 156 156 255 255 255 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 159, Max flush: 258 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.750532387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.795120617Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.795818846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.795861969Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.795877547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.795893085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} {"msg":"Crucible d134d55f-d059-467c-ae68-bb0f2a61f86e has session id: 88c3fb9c-4f69-4705-9e11-91a03820c814","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.796565637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Upstairs opts: Upstairs UUID: d134d55f-d059-467c-ae68-bb0f2a61f86e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.796661763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Crucible stats registered with UUID: d134d55f-d059-467c-ae68-bb0f2a61f86e","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.796693448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:02.796715545Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797131085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797155141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} {"msg":"d134d55f-d059-467c-ae68-bb0f2a61f86e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797220812Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797239249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797262716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797286352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797313698Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797342034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.79737189Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797398606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797427601Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797455168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"ds_connection connected from Ok(127.0.0.1:38087)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797483684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"ds_connection connected from Ok(127.0.0.1:39059)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797661498Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"ds_connection connected from Ok(127.0.0.1:54538)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.797698432Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.798171863Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.798202499Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.798365725Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.79880902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.798837566Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799064403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799375478Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799401214Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[0]R flush_numbers[0..12]: [247, 253, 258, 258, 255, 255, 256, 255, 256, 257, 256, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799627491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[0]R generation[0..12]: [150, 154, 159, 159, 156, 156, 158, 156, 158, 158, 158, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799647858Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799662186Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[1]R flush_numbers[0..12]: [247, 253, 258, 258, 255, 255, 256, 255, 256, 257, 256, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799676014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[1]R generation[0..12]: [150, 154, 159, 159, 156, 156, 158, 156, 158, 158, 158, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799689642Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.7997086Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[2]R flush_numbers[0..12]: [247, 253, 258, 258, 255, 255, 256, 255, 256, 257, 256, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799723407Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[2]R generation[0..12]: [150, 154, 159, 159, 156, 156, 158, 156, 158, 158, 158, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799737085Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799751573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799764601Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Generation requested: 160 >= found:160","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.79984062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799870226Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799885154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"":"downstairs","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799898682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.79991055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"0","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799924058Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"1","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799937066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"client":"2","":"downstairs","session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799950134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799962253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"d134d55f-d059-467c-ae68-bb0f2a61f86e is now active with session: 88c3fb9c-4f69-4705-9e11-91a03820c814","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799975031Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.799991618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822,"session_id":"88c3fb9c-4f69-4705-9e11-91a03820c814"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:02.800004346Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 12288 data: 14 15 17 02/30 Write block 200 len 24576 data: 14 13 14 10 8 8 03/30 Write block 221 len 12288 data: 11 12 12 04/30 Flush 05/30 Write block 290 len 12288 data: 16 15 13 06/30 Write block 175 len 8192 data: 9 10 07/30 Write block 373 len 16384 data: 8 9 11 10 08/30 Write block 500 len 28672 data: 14 14 13 17 17 17 17 09/30 Read block 505 len 8192 10/30 Flush 11/30 Write block 110 len 20480 data: 10 11 10 11 9 12/30 Read block 79 len 40960 13/30 Read block 89 len 20480 14/30 Read block 530 len 28672 15/30 Write block 199 len 16384 data: 14 15 14 15 16/30 Read block 587 len 24576 17/30 Read block 543 len 36864 18/30 Read block 79 len 16384 19/30 Write block 121 len 12288 data: 8 6 8 20/30 Flush 21/30 Write block 576 len 16384 data: 17 15 12 10 22/30 Write block 416 len 40960 data: 12 10 13 15 15 15 17 14 13 14 23/30 Read block 332 len 4096 24/30 Read block 346 len 40960 25/30 Read block 343 len 8192 26/30 Read block 59 len 28672 27/30 Read block 47 len 40960 28/30 Write block 78 len 24576 data: 10 11 11 8 6 7 29/30 Read block 178 len 20480 30/30 Read block 338 len 36864 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:02.819506888Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 159 159 159 258 258 258 T T T 4 080-099 156 156 156 255 255 255 T T T 5 100-119 160 160 160 261 261 261 F F F 6 120-139 160 160 160 261 261 261 F F F 7 140-159 156 156 156 255 255 255 F F F 8 160-179 160 160 160 260 260 260 F F F 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 160 160 160 259 259 259 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 160 160 160 260 260 260 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 159 159 159 258 258 258 F F F 20 400-419 156 156 156 255 255 255 T T T 21 420-439 151 151 151 251 251 251 T T T 22 440-459 156 156 156 255 255 255 F F F 23 460-479 156 156 156 255 255 255 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 158 158 158 256 256 256 F F F 28 560-579 156 156 156 255 255 255 T T T 29 580-599 156 156 156 255 255 255 F F F Max gen: 160, Max flush: 261 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 161 161 161 262 262 262 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 160 160 160 261 261 261 F F F 7 140-159 156 156 156 255 255 255 F F F 8 160-179 160 160 160 260 260 260 F F F 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 160 160 160 259 259 259 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 152 152 152 252 252 252 F F F 14 280-299 160 160 160 260 260 260 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 155 155 155 254 254 254 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 159 159 159 258 258 258 F F F 20 400-419 161 161 161 262 262 262 F F F 21 420-439 161 161 161 262 262 262 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 156 156 156 255 255 255 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 158 158 158 256 256 256 F F F 28 560-579 161 161 161 262 262 262 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 161, Max flush: 262 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.057104388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.101213509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.101970219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.102021071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.102061105Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.102080623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} {"msg":"Crucible 5a59e23f-535d-49f7-ba57-e49eb2883536 has session id: b7d762b3-f209-42bc-b726-71ade34f0c1c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.10271463Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Upstairs opts: Upstairs UUID: 5a59e23f-535d-49f7-ba57-e49eb2883536, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.102756334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Crucible stats registered with UUID: 5a59e23f-535d-49f7-ba57-e49eb2883536","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.102779891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:07.102800638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103260921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103295676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} {"msg":"5a59e23f-535d-49f7-ba57-e49eb2883536 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103391742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103426637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103459032Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103497747Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103521993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.10354173Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103562597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103589193Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.10361287Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52392)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103636406Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49560)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103659393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63430)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.103683759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.104308758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.104336734Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.104557822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105048561Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105078146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105093234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105570815Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105681399Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[0]R flush_numbers[0..12]: [247, 253, 258, 262, 262, 261, 261, 255, 260, 261, 261, 259]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.10574438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[0]R generation[0..12]: [150, 154, 159, 161, 161, 160, 160, 156, 160, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105775495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105791573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[1]R flush_numbers[0..12]: [247, 253, 258, 262, 262, 261, 261, 255, 260, 261, 261, 259]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105805671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[1]R generation[0..12]: [150, 154, 159, 161, 161, 160, 160, 156, 160, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105819358Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105832956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[2]R flush_numbers[0..12]: [247, 253, 258, 262, 262, 261, 261, 255, 260, 261, 261, 259]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105846255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[2]R generation[0..12]: [150, 154, 159, 161, 161, 160, 160, 156, 160, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105859653Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105873281Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105886129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105898917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105911485Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105929362Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"":"downstairs","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105942571Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105956259Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"0","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105969307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"1","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105981935Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"client":"2","":"downstairs","session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.105994783Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.106006801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"5a59e23f-535d-49f7-ba57-e49eb2883536 is now active with session: b7d762b3-f209-42bc-b726-71ade34f0c1c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.106019469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.106031228Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830,"session_id":"b7d762b3-f209-42bc-b726-71ade34f0c1c"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:07.106043196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 4096 data: 17 02/30 Write block 383 len 32768 data: 11 9 12 12 11 15 16 18 03/30 Write block 308 len 40960 data: 9 10 9 9 14 14 12 12 10 11 04/30 Read block 243 len 16384 05/30 Read block 325 len 36864 06/30 Write block 12 len 20480 data: 9 6 8 9 5 07/30 Read block 541 len 4096 08/30 Write block 471 len 16384 data: 12 14 14 14 09/30 Write block 400 len 4096 data: 18 10/30 Flush 11/30 Write block 274 len 36864 data: 13 14 14 14 12 11 12 14 13 12/30 Write block 307 len 36864 data: 9 10 11 10 10 15 15 13 13 13/30 Read block 163 len 32768 14/30 Read block 95 len 32768 15/30 Read block 539 len 32768 16/30 Read block 246 len 12288 17/30 Read block 155 len 20480 18/30 Read block 188 len 36864 19/30 Read block 7 len 24576 20/30 Read block 279 len 4096 21/30 Write block 225 len 20480 data: 10 12 11 8 10 22/30 Read block 254 len 12288 23/30 Read block 254 len 4096 24/30 Flush 25/30 Write block 540 len 16384 data: 12 12 12 13 26/30 Write block 294 len 24576 data: 16 18 14 12 10 10 27/30 Read block 338 len 16384 28/30 Write block 384 len 12288 data: 10 13 13 29/30 Write block 138 len 12288 data: 8 9 9 30/30 Write block 258 len 8192 data: 9 10 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:07.123917015Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 263 263 263 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 161 161 161 262 262 262 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 160 160 160 261 261 261 T T T 7 140-159 156 156 156 255 255 255 T T T 8 160-179 160 160 160 260 260 260 F F F 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 162 162 162 264 264 264 F F F 12 240-259 154 154 154 253 253 253 T T T 13 260-279 162 162 162 264 264 264 F F F 14 280-299 162 162 162 264 264 264 T T T 15 300-319 162 162 162 264 264 264 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 162 162 162 263 263 263 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 162 162 162 263 263 263 T T T 20 400-419 162 162 162 263 263 263 F F F 21 420-439 161 161 161 262 262 262 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 162 162 162 263 263 263 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 158 158 158 256 256 256 T T T 28 560-579 161 161 161 262 262 262 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 162, Max flush: 264 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 263 263 263 F F F 1 020-039 154 154 154 253 253 253 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 161 161 161 262 262 262 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 163 163 163 265 265 265 F F F 7 140-159 163 163 163 265 265 265 F F F 8 160-179 160 160 160 260 260 260 F F F 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 162 162 162 264 264 264 F F F 12 240-259 163 163 163 265 265 265 F F F 13 260-279 162 162 162 264 264 264 F F F 14 280-299 163 163 163 265 265 265 F F F 15 300-319 162 162 162 264 264 264 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 162 162 162 263 263 263 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 163 163 163 265 265 265 F F F 20 400-419 162 162 162 263 263 263 F F F 21 420-439 161 161 161 262 262 262 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 162 162 162 263 263 263 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 163 163 163 265 265 265 F F F 28 560-579 161 161 161 262 262 262 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 163, Max flush: 265 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.371396856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.416349204Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417021826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.41706629Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417082657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417097675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} {"msg":"Crucible a89c7078-f342-4a7e-b899-58a03eed5d18 has session id: 656c5422-bce7-4efd-9ec7-898bac6af9fc","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417661783Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Upstairs opts: Upstairs UUID: a89c7078-f342-4a7e-b899-58a03eed5d18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417705447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Crucible stats registered with UUID: a89c7078-f342-4a7e-b899-58a03eed5d18","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.417729133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:11.41774996Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418202664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.41823059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} {"msg":"a89c7078-f342-4a7e-b899-58a03eed5d18 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418292701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418311429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418344664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418361491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418381638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418407404Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.41843623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418463616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418569891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418599716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34189)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418698302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:32980)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418737327Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59068)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.418761553Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.419291396Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.419335129Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.419418247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.419986935Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420036487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420071722Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.42063776Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420663326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[0]R flush_numbers[0..12]: [263, 253, 258, 262, 262, 261, 265, 265, 260, 261, 261, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420688983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[0]R generation[0..12]: [162, 154, 159, 161, 161, 160, 163, 163, 160, 160, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.42070402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420718288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[1]R flush_numbers[0..12]: [263, 253, 258, 262, 262, 261, 265, 265, 260, 261, 261, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420732496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[1]R generation[0..12]: [162, 154, 159, 161, 161, 160, 163, 163, 160, 160, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420746194Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420759902Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[2]R flush_numbers[0..12]: [263, 253, 258, 262, 262, 261, 265, 265, 260, 261, 261, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.42077364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[2]R generation[0..12]: [162, 154, 159, 161, 161, 160, 163, 163, 160, 160, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420787288Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420800936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420818394Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420831982Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.42084477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420857398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"":"downstairs","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-14T16:17:11.420870216Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420892573Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"0","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420905561Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"1","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420918229Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"client":"2","":"downstairs","session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420931037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420942935Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"a89c7078-f342-4a7e-b899-58a03eed5d18 is now active with session: 656c5422-bce7-4efd-9ec7-898bac6af9fc","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420955474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.420967212Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838,"session_id":"656c5422-bce7-4efd-9ec7-898bac6af9fc"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:11.42097905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 32768 data: 13 14 13 10 11 9 11 9 02/30 Flush 03/30 Flush 04/30 Write block 286 len 24576 data: 15 14 14 17 17 16 05/30 Read block 564 len 24576 06/30 Write block 229 len 16384 data: 11 11 11 16 07/30 Read block 188 len 28672 08/30 Read block 529 len 32768 09/30 Write block 553 len 8192 data: 12 10 10/30 Flush 11/30 Write block 9 len 4096 data: 11 12/30 Write block 302 len 12288 data: 9 9 10 13/30 Read block 301 len 40960 14/30 Write block 347 len 40960 data: 16 15 16 15 15 13 14 15 14 14 15/30 Write block 157 len 40960 data: 13 11 10 12 13 11 9 10 12 11 16/30 Read block 48 len 12288 17/30 Read block 592 len 24576 18/30 Write block 73 len 28672 data: 12 10 11 10 12 11 12 19/30 Write block 21 len 8192 data: 4 4 20/30 Read block 325 len 28672 21/30 Write block 395 len 24576 data: 13 15 18 19 21 19 22/30 Write block 121 len 36864 data: 9 7 9 7 5 4 5 8 11 23/30 Read block 141 len 32768 24/30 Read block 598 len 8192 25/30 Flush 26/30 Write block 169 len 20480 data: 10 8 9 9 13 27/30 Read block 295 len 8192 28/30 Read block 142 len 28672 29/30 Write block 130 len 24576 data: 12 11 12 14 14 10 30/30 Write block 168 len 28672 data: 7 11 9 10 10 14 12 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:11.440894682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 269 269 269 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 164 164 164 269 269 269 T T T 7 140-159 164 164 164 269 269 269 F F F 8 160-179 164 164 164 269 269 269 T T T 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 163 163 163 265 265 265 F F F 13 260-279 162 162 162 264 264 264 F F F 14 280-299 164 164 164 268 268 268 F F F 15 300-319 164 164 164 269 269 269 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 164 164 164 269 269 269 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 164 164 164 269 269 269 F F F 20 400-419 164 164 164 269 269 269 F F F 21 420-439 161 161 161 262 262 262 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 162 162 162 263 263 263 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 164 164 164 268 268 268 F F F 28 560-579 161 161 161 262 262 262 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 164, Max flush: 269 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 269 269 269 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 165 165 165 270 270 270 F F F 7 140-159 164 164 164 269 269 269 F F F 8 160-179 165 165 165 270 270 270 F F F 9 180-199 160 160 160 261 261 261 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 163 163 163 265 265 265 F F F 13 260-279 162 162 162 264 264 264 F F F 14 280-299 164 164 164 268 268 268 F F F 15 300-319 164 164 164 269 269 269 F F F 16 320-339 160 160 160 259 259 259 F F F 17 340-359 164 164 164 269 269 269 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 164 164 164 269 269 269 F F F 20 400-419 164 164 164 269 269 269 F F F 21 420-439 161 161 161 262 262 262 F F F 22 440-459 156 156 156 255 255 255 F F F 23 460-479 162 162 162 263 263 263 F F F 24 480-499 156 156 156 255 255 255 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 164 164 164 268 268 268 F F F 28 560-579 161 161 161 262 262 262 F F F 29 580-599 156 156 156 255 255 255 F F F Max gen: 165, Max flush: 270 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.667960375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.712084383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.712799979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.712841933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.71286086Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.712886766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} {"msg":"Crucible 93bd4db7-8a07-4dba-9e74-ff0eb3f27c7a has session id: abb846ba-356a-414b-9bd5-9a37fb2aceb0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.713509256Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Upstairs opts: Upstairs UUID: 93bd4db7-8a07-4dba-9e74-ff0eb3f27c7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.713543581Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Crucible stats registered with UUID: 93bd4db7-8a07-4dba-9e74-ff0eb3f27c7a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.713559219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:15.713575616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714126836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714158591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} {"msg":"93bd4db7-8a07-4dba-9e74-ff0eb3f27c7a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714220382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714244979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714264866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714278994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714293542Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714309999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714335995Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714360252Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714387788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714458388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36876)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714506471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40523)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714533427Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52762)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.714686395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.715189671Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.715228925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.715876961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.715906637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.716010662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.716562462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.716599836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.716632381Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[0]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 270, 269, 270, 261, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717186391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[0]R generation[0..12]: [164, 164, 159, 164, 161, 160, 165, 164, 165, 160, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717214476Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717230124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[1]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 270, 269, 270, 261, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717244442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[1]R generation[0..12]: [164, 164, 159, 164, 161, 160, 165, 164, 165, 160, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.71725804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717275218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[2]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 270, 269, 270, 261, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717289196Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[2]R generation[0..12]: [164, 164, 159, 164, 161, 160, 165, 164, 165, 160, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717302614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717316182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.71732878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717341758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717359486Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717372213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"":"downstairs","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717384882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.71739648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"0","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717409428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"1","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717422486Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"client":"2","":"downstairs","session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717435834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717447882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"93bd4db7-8a07-4dba-9e74-ff0eb3f27c7a is now active with session: abb846ba-356a-414b-9bd5-9a37fb2aceb0","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717461101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717474789Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846,"session_id":"abb846ba-356a-414b-9bd5-9a37fb2aceb0"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:15.717486927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 24576 data: 19 17 18 16 13 14 02/30 Flush 03/30 Write block 578 len 32768 data: 13 11 9 11 12 8 12 13 04/30 Write block 346 len 4096 data: 18 05/30 Read block 392 len 28672 06/30 Write block 453 len 40960 data: 15 15 15 13 14 14 17 17 14 16 07/30 Write block 436 len 40960 data: 13 10 9 8 8 11 12 15 15 13 08/30 Flush 09/30 Write block 316 len 40960 data: 11 12 9 7 4 6 8 11 14 15 10/30 Read block 137 len 4096 11/30 Read block 489 len 36864 12/30 Write block 485 len 36864 data: 11 10 10 11 13 11 10 9 6 13/30 Write block 163 len 4096 data: 10 14/30 Read block 483 len 16384 15/30 Write block 471 len 16384 data: 13 15 15 15 16/30 Read block 324 len 28672 17/30 Read block 380 len 36864 18/30 Read block 4 len 4096 19/30 Read block 228 len 4096 20/30 Read block 397 len 20480 21/30 Write block 408 len 4096 data: 15 22/30 Read block 344 len 20480 23/30 Read block 177 len 28672 24/30 Write block 121 len 4096 data: 10 25/30 Write block 182 len 12288 data: 7 10 13 26/30 Write block 502 len 32768 data: 14 18 18 18 18 22 21 17 27/30 Read block 353 len 12288 28/30 Read block 402 len 4096 29/30 Read block 548 len 8192 30/30 Read block 4 len 8192 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 10 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:15.73642774Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 269 269 269 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 165 165 165 270 270 270 T T T 7 140-159 164 164 164 269 269 269 F F F 8 160-179 165 165 165 270 270 270 T T T 9 180-199 160 160 160 261 261 261 T T T 10 200-219 160 160 160 261 261 261 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 163 163 163 265 265 265 F F F 13 260-279 162 162 162 264 264 264 F F F 14 280-299 164 164 164 268 268 268 F F F 15 300-319 164 164 164 269 269 269 T T T 16 320-339 160 160 160 259 259 259 T T T 17 340-359 166 166 166 272 272 272 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 164 164 164 269 269 269 T T T 21 420-439 166 166 166 272 272 272 F F F 22 440-459 166 166 166 272 272 272 F F F 23 460-479 166 166 166 272 272 272 T T T 24 480-499 156 156 156 255 255 255 T T T 25 500-519 160 160 160 260 260 260 T T T 26 520-539 159 159 159 258 258 258 F F F 27 540-559 164 164 164 268 268 268 F F F 28 560-579 166 166 166 272 272 272 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 166, Max flush: 272 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 269 269 269 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 160 160 160 261 261 261 F F F 6 120-139 167 167 167 273 273 273 F F F 7 140-159 164 164 164 269 269 269 F F F 8 160-179 167 167 167 273 273 273 F F F 9 180-199 167 167 167 273 273 273 F F F 10 200-219 160 160 160 261 261 261 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 163 163 163 265 265 265 F F F 13 260-279 162 162 162 264 264 264 F F F 14 280-299 164 164 164 268 268 268 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 160 160 160 260 260 260 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 166 166 166 272 272 272 F F F 22 440-459 166 166 166 272 272 272 F F F 23 460-479 167 167 167 273 273 273 F F F 24 480-499 167 167 167 273 273 273 F F F 25 500-519 167 167 167 273 273 273 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 164 164 164 268 268 268 F F F 28 560-579 166 166 166 272 272 272 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 167, Max flush: 273 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:19.993985344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.037618094Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.03826606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.038310073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.038326641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.038341709Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} {"msg":"Crucible 3bff78c8-433f-4b46-bf00-31c14691ba50 has session id: 212c9215-b827-48e5-85c4-4d3bd4c2d92a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.03894878Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Upstairs opts: Upstairs UUID: 3bff78c8-433f-4b46-bf00-31c14691ba50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.038985855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Crucible stats registered with UUID: 3bff78c8-433f-4b46-bf00-31c14691ba50","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.03901684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:20.039030948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039482952Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039505979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} {"msg":"3bff78c8-433f-4b46-bf00-31c14691ba50 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.03956584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039582778Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039606124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039620392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.03963819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039658317Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039684983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039711769Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039729127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039742845Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34456)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039769511Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36235)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039795917Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55290)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.039821873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.040476918Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.040503624Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.041134442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.041165098Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.041735335Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.041760161Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.042296823Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.042765965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[0]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 273, 269, 273, 273, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043261992Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[0]R generation[0..12]: [164, 164, 159, 164, 161, 160, 167, 164, 167, 167, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043290758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043309235Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[1]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 273, 269, 273, 273, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043327313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[1]R generation[0..12]: [164, 164, 159, 164, 161, 160, 167, 164, 167, 167, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.04334662Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043377616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[2]R flush_numbers[0..12]: [269, 269, 258, 269, 262, 261, 273, 269, 273, 273, 261, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043411771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[2]R generation[0..12]: [164, 164, 159, 164, 161, 160, 167, 164, 167, 167, 160, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043439836Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043461103Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:17:20.043489839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043521295Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043557119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043576427Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"":"downstairs","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043604423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043632108Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"0","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043662804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"1","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043680801Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"client":"2","":"downstairs","session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043707728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043735393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"3bff78c8-433f-4b46-bf00-31c14691ba50 is now active with session: 212c9215-b827-48e5-85c4-4d3bd4c2d92a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043765509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043787066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854,"session_id":"212c9215-b827-48e5-85c4-4d3bd4c2d92a"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:20.043811052Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 204 len 28672 data: 9 9 8 9 12 16 13 02/30 Write block 112 len 24576 data: 11 12 10 8 7 6 03/30 Read block 248 len 40960 04/30 Read block 439 len 36864 05/30 Read block 447 len 24576 06/30 Read block 426 len 12288 07/30 Read block 466 len 20480 08/30 Write block 248 len 36864 data: 11 13 11 9 10 11 13 10 11 09/30 Flush 10/30 Write block 240 len 8192 data: 18 16 11/30 Read block 139 len 20480 12/30 Write block 118 len 40960 data: 7 6 8 11 8 10 8 6 5 6 13/30 Flush 14/30 Write block 548 len 40960 data: 15 14 11 12 10 13 11 13 16 15 15/30 Flush 16/30 Write block 6 len 28672 data: 8 10 10 12 13 10 10 17/30 Write block 548 len 28672 data: 16 15 12 13 11 14 12 18/30 Flush 19/30 Write block 173 len 8192 data: 15 13 20/30 Read block 438 len 8192 21/30 Read block 325 len 28672 22/30 Write block 123 len 4096 data: 11 23/30 Read block 0 len 32768 24/30 Write block 274 len 40960 data: 14 15 15 15 13 12 13 15 14 13 25/30 Write block 431 len 28672 data: 16 14 12 11 15 14 11 26/30 Read block 520 len 24576 27/30 Write block 289 len 12288 data: 18 18 17 28/30 Read block 535 len 36864 29/30 Read block 102 len 32768 30/30 Write block 366 len 20480 data: 14 14 14 14 14 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:20.063519744Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 277 277 277 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 168 168 168 275 275 275 F F F 6 120-139 168 168 168 275 275 275 T T T 7 140-159 164 164 164 269 269 269 F F F 8 160-179 167 167 167 273 273 273 T T T 9 180-199 167 167 167 273 273 273 F F F 10 200-219 168 168 168 274 274 274 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 162 162 162 264 264 264 T T T 14 280-299 164 164 164 268 268 268 T T T 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 160 160 160 260 260 260 T T T 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 166 166 166 272 272 272 T T T 22 440-459 166 166 166 272 272 272 F F F 23 460-479 167 167 167 273 273 273 F F F 24 480-499 167 167 167 273 273 273 F F F 25 500-519 167 167 167 273 273 273 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 168 168 168 277 277 277 F F F 28 560-579 166 166 166 272 272 272 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 168, Max flush: 277 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 277 277 277 F F F 1 020-039 164 164 164 269 269 269 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 168 168 168 275 275 275 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 164 164 164 269 269 269 F F F 8 160-179 169 169 169 278 278 278 F F F 9 180-199 167 167 167 273 273 273 F F F 10 200-219 168 168 168 274 274 274 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 169 169 169 278 278 278 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 169 169 169 278 278 278 F F F 22 440-459 166 166 166 272 272 272 F F F 23 460-479 167 167 167 273 273 273 F F F 24 480-499 167 167 167 273 273 273 F F F 25 500-519 167 167 167 273 273 273 F F F 26 520-539 159 159 159 258 258 258 F F F 27 540-559 168 168 168 277 277 277 F F F 28 560-579 166 166 166 272 272 272 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 169, Max flush: 278 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.313430462Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:24.357525704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.357881153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.389377479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.389935148Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.389976752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.389993159Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.390010517Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} {"msg":"Crucible b1d4aa7f-c84f-493d-a8e0-f5197c0801b1 has session id: 247ee118-d962-4440-9bb7-06148674c359","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.390598851Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Upstairs opts: Upstairs UUID: b1d4aa7f-c84f-493d-a8e0-f5197c0801b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.390643024Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Crucible stats registered with UUID: b1d4aa7f-c84f-493d-a8e0-f5197c0801b1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.390667271Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:24.390680819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391049705Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391074122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} {"msg":"b1d4aa7f-c84f-493d-a8e0-f5197c0801b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391143632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391161669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391184596Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391198494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391212602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.39122579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391239948Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391253096Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391267444Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391280672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"ds_connection connected from Ok(127.0.0.1:52833)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391389766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"ds_connection connected from Ok(127.0.0.1:37842)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391426101Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"ds_connection connected from Ok(127.0.0.1:35689)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.391606814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392128728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392157934Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392182911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392710814Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392748688Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.392887638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393297138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393389465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[0]R flush_numbers[0..12]: [277, 269, 258, 269, 262, 275, 278, 269, 278, 273, 274, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.39349528Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[0]R generation[0..12]: [168, 164, 159, 164, 161, 168, 169, 164, 169, 167, 168, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393514717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393528855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[1]R flush_numbers[0..12]: [277, 269, 258, 269, 262, 275, 278, 269, 278, 273, 274, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393542893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[1]R generation[0..12]: [168, 164, 159, 164, 161, 168, 169, 164, 169, 167, 168, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.39356107Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393575238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[2]R flush_numbers[0..12]: [277, 269, 258, 269, 262, 275, 278, 269, 278, 273, 274, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393589066Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[2]R generation[0..12]: [168, 164, 159, 164, 161, 168, 169, 164, 169, 167, 168, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393602484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393616202Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.39362885Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393641738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393654356Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393672874Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"":"downstairs","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393685882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.39369771Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"0","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393710628Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"1","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393723736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"client":"2","":"downstairs","session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393736604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393748503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"b1d4aa7f-c84f-493d-a8e0-f5197c0801b1 is now active with session: 247ee118-d962-4440-9bb7-06148674c359","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393763341Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393775269Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862,"session_id":"247ee118-d962-4440-9bb7-06148674c359"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:24.393787087Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 465 len 16384 data: 15 15 12 13 03/30 Write block 196 len 8192 data: 15 14 04/30 Write block 34 len 20480 data: 15 12 11 11 14 05/30 Write block 147 len 24576 data: 17 21 23 16 15 13 06/30 Read block 313 len 20480 07/30 Read block 470 len 32768 08/30 Read block 437 len 28672 09/30 Write block 366 len 20480 data: 15 15 15 15 15 10/30 Write block 161 len 8192 data: 14 12 11/30 Write block 213 len 4096 data: 8 12/30 Read block 508 len 24576 13/30 Flush 14/30 Write block 572 len 20480 data: 16 18 18 17 18 15/30 Read block 214 len 28672 16/30 Flush 17/30 Write block 1 len 20480 data: 6 7 8 9 8 18/30 Write block 514 len 4096 data: 9 19/30 Read block 329 len 40960 20/30 Read block 211 len 4096 21/30 Write block 438 len 36864 data: 10 9 9 12 13 16 16 14 11 22/30 Read block 171 len 28672 23/30 Read block 198 len 36864 24/30 Flush 25/30 Write block 448 len 36864 data: 12 11 12 14 12 16 16 16 14 26/30 Read block 294 len 24576 27/30 Read block 303 len 12288 28/30 Read block 495 len 36864 29/30 Write block 458 len 12288 data: 15 18 18 30/30 Write block 534 len 32768 data: 11 14 15 17 15 14 13 13 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:24.412644973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 168 168 168 275 275 275 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 170 170 170 280 280 280 F F F 8 160-179 170 170 170 280 280 280 F F F 9 180-199 170 170 170 280 280 280 F F F 10 200-219 170 170 170 280 280 280 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 170 170 170 282 282 282 F F F 22 440-459 170 170 170 282 282 282 T T T 23 460-479 170 170 170 280 280 280 T T T 24 480-499 167 167 167 273 273 273 F F F 25 500-519 170 170 170 282 282 282 F F F 26 520-539 159 159 159 258 258 258 T T T 27 540-559 168 168 168 277 277 277 T T T 28 560-579 170 170 170 281 281 281 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 170, Max flush: 282 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 159 159 159 258 258 258 F F F 3 060-079 164 164 164 269 269 269 F F F 4 080-099 161 161 161 262 262 262 F F F 5 100-119 168 168 168 275 275 275 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 170 170 170 280 280 280 F F F 8 160-179 170 170 170 280 280 280 F F F 9 180-199 170 170 170 280 280 280 F F F 10 200-219 170 170 170 280 280 280 F F F 11 220-239 164 164 164 268 268 268 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 170 170 170 282 282 282 F F F 22 440-459 171 171 171 283 283 283 F F F 23 460-479 171 171 171 283 283 283 F F F 24 480-499 167 167 167 273 273 273 F F F 25 500-519 170 170 170 282 282 282 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 171 171 171 283 283 283 F F F 28 560-579 170 170 170 281 281 281 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 171, Max flush: 283 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.672130447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:28.715495136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.715910395Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.748281604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.748849991Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.748884976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.748898554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.748912762Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} {"msg":"Crucible a1b46b9d-b202-4b57-ac6e-32d88d9c63d6 has session id: b9ea529c-02a6-4864-a99d-3890edecf693","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.749576665Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Upstairs opts: Upstairs UUID: a1b46b9d-b202-4b57-ac6e-32d88d9c63d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.749620689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Crucible stats registered with UUID: a1b46b9d-b202-4b57-ac6e-32d88d9c63d6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.749644826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:28.749666012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750072063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.75009592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} {"msg":"a1b46b9d-b202-4b57-ac6e-32d88d9c63d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750157911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750174618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750196725Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750210533Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750224691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750241559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750256206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.75029855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750327796Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750348423Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"ds_connection connected from Ok(127.0.0.1:52153)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750362731Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"ds_connection connected from Ok(127.0.0.1:62668)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750621763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"ds_connection connected from Ok(127.0.0.1:50076)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.750661758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.751348598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.751506404Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.75153908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752046786Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.75208827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752187605Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752564131Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752654807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[0]R flush_numbers[0..12]: [282, 280, 258, 269, 262, 275, 278, 280, 280, 280, 280, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752761192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[0]R generation[0..12]: [170, 170, 159, 164, 161, 168, 169, 170, 170, 170, 170, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752781139Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752795767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[1]R flush_numbers[0..12]: [282, 280, 258, 269, 262, 275, 278, 280, 280, 280, 280, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752809465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[1]R generation[0..12]: [170, 170, 159, 164, 161, 168, 169, 170, 170, 170, 170, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752822933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752836501Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[2]R flush_numbers[0..12]: [282, 280, 258, 269, 262, 275, 278, 280, 280, 280, 280, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752849819Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[2]R generation[0..12]: [170, 170, 159, 164, 161, 168, 169, 170, 170, 170, 170, 164]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752863127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752876855Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752889673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752902421Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:17:28.752914979Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752942955Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"":"downstairs","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752967872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.752988619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"0","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.753012305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"1","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.753033822Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"client":"2","":"downstairs","session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.75304791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.753060138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"a1b46b9d-b202-4b57-ac6e-32d88d9c63d6 is now active with session: b9ea529c-02a6-4864-a99d-3890edecf693","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.753080885Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-14T16:17:28.753102492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870,"session_id":"b9ea529c-02a6-4864-a99d-3890edecf693"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:28.753126759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 36864 data: 13 15 13 11 11 13 12 9 8 02/30 Read block 523 len 32768 03/30 Read block 590 len 40960 04/30 Flush 05/30 Write block 60 len 28672 data: 8 11 15 20 18 22 17 06/30 Read block 400 len 8192 07/30 Write block 447 len 4096 data: 14 08/30 Read block 245 len 40960 09/30 Read block 23 len 40960 10/30 Write block 179 len 20480 data: 10 10 7 8 11 11/30 Write block 178 len 28672 data: 10 11 11 8 9 12 14 12/30 Read block 12 len 36864 13/30 Read block 447 len 24576 14/30 Write block 436 len 24576 data: 15 12 11 10 10 13 15/30 Write block 49 len 24576 data: 11 9 12 12 12 10 16/30 Read block 385 len 32768 17/30 Read block 355 len 4096 18/30 Write block 555 len 32768 data: 14 17 16 15 15 12 10 10 19/30 Write block 335 len 16384 data: 16 15 16 18 20/30 Read block 238 len 12288 21/30 Read block 178 len 32768 22/30 Read block 408 len 12288 23/30 Read block 567 len 4096 24/30 Write block 479 len 24576 data: 11 10 10 9 9 10 25/30 Write block 222 len 12288 data: 13 13 11 26/30 Read block 20 len 32768 27/30 Read block 513 len 36864 28/30 Write block 542 len 12288 data: 13 14 12 29/30 Read block 309 len 40960 30/30 Write block 83 len 28672 data: 8 5 7 8 9 9 9 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 7 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:28.773204037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 159 159 159 258 258 258 T T T 3 060-079 164 164 164 269 269 269 T T T 4 080-099 161 161 161 262 262 262 T T T 5 100-119 168 168 168 275 275 275 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 170 170 170 280 280 280 F F F 8 160-179 172 172 172 284 284 284 T T T 9 180-199 170 170 170 280 280 280 T T T 10 200-219 170 170 170 280 280 280 F F F 11 220-239 164 164 164 268 268 268 T T T 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 167 167 167 273 273 273 T T T 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 170 170 170 282 282 282 T T T 22 440-459 171 171 171 283 283 283 T T T 23 460-479 171 171 171 283 283 283 T T T 24 480-499 167 167 167 273 273 273 T T T 25 500-519 170 170 170 282 282 282 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 171 171 171 283 283 283 T T T 28 560-579 170 170 170 281 281 281 T T T 29 580-599 166 166 166 272 272 272 F F F Max gen: 172, Max flush: 284 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 173 173 173 285 285 285 F F F 3 060-079 173 173 173 285 285 285 F F F 4 080-099 173 173 173 285 285 285 F F F 5 100-119 168 168 168 275 275 275 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 170 170 170 280 280 280 F F F 8 160-179 173 173 173 285 285 285 F F F 9 180-199 173 173 173 285 285 285 F F F 10 200-219 170 170 170 280 280 280 F F F 11 220-239 173 173 173 285 285 285 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 F F F 16 320-339 173 173 173 285 285 285 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 173 173 173 285 285 285 F F F 22 440-459 173 173 173 285 285 285 F F F 23 460-479 173 173 173 285 285 285 F F F 24 480-499 173 173 173 285 285 285 F F F 25 500-519 170 170 170 282 282 282 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 173 173 173 285 285 285 F F F 29 580-599 166 166 166 272 272 272 F F F Max gen: 173, Max flush: 285 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.086789358Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.131623352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.132217706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.132266649Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.132284416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.132302383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} {"msg":"Crucible 8afe574e-abac-4496-af4f-7d5f7fa83bc6 has session id: cb03040f-b2ab-4c7e-acdd-39745adce8e2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.132975816Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Upstairs opts: Upstairs UUID: 8afe574e-abac-4496-af4f-7d5f7fa83bc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.13301118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Crucible stats registered with UUID: 8afe574e-abac-4496-af4f-7d5f7fa83bc6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133037827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:33.133053594Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133434259Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133458755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} {"msg":"8afe574e-abac-4496-af4f-7d5f7fa83bc6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133517187Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133535314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133559081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133573618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133590626Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133614063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133657446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133687922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133713178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133727556Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44083)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133741174Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64880)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.133936296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50733)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.13397351Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.134452391Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.134495224Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.13452706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135126503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135154308Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135216679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135756831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135782657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[0]R flush_numbers[0..12]: [282, 280, 285, 285, 285, 275, 278, 280, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135796265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[0]R generation[0..12]: [170, 170, 173, 173, 173, 168, 169, 170, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135810413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135825491Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[1]R flush_numbers[0..12]: [282, 280, 285, 285, 285, 275, 278, 280, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135839749Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[1]R generation[0..12]: [170, 170, 173, 173, 173, 168, 169, 170, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135856716Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135871564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[2]R flush_numbers[0..12]: [282, 280, 285, 285, 285, 275, 278, 280, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135885122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[2]R generation[0..12]: [170, 170, 173, 173, 173, 168, 169, 170, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.13589876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135912508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135925226Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135938154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135950503Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.135963151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B"Wait for a query_work_queue command to finish before sending IO ,"pid":1878,"":"downstairs","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136043309Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136065586Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"0","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136080134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"1","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136093962Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"client":"2","":"downstairs","session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.13610707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136122707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"8afe574e-abac-4496-af4f-7d5f7fa83bc6 is now active with session: cb03040f-b2ab-4c7e-acdd-39745adce8e2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136136006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136148014Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878,"session_id":"cb03040f-b2ab-4c7e-acdd-39745adce8e2"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:33.136160002Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 36864 data: 15 13 11 12 12 13 11 9 8 02/30 Read block 255 len 36864 03/30 Write block 580 len 24576 data: 10 12 13 9 13 14 04/30 Read block 465 len 24576 05/30 Read block 226 len 8192 06/30 Read block 510 len 24576 07/30 Read block 455 len 28672 08/30 Read block 469 len 12288 09/30 Read block 355 len 24576 10/30 Flush 11/30 Write block 488 len 8192 data: 12 14 12/30 Write block 510 len 32768 data: 14 12 11 10 10 9 9 11 13/30 Write block 332 len 20480 data: 21 18 18 17 16 14/30 Write block 147 len 24576 data: 18 22 24 17 16 14 15/30 Write block 56 len 32768 data: 10 12 11 8 9 12 16 21 16/30 Flush 17/30 Write block 99 len 8192 data: 16 17 18/30 Write block 36 len 40960 data: 12 12 15 16 17 13 12 12 9 11 19/30 Read block 35 len 4096 20/30 Read block 359 len 40960 21/30 Read block 97 len 4096 22/30 Write block 315 len 4096 data: 14 23/30 Read block 233 len 4096 24/30 Read block 247 len 24576 25/30 Read block 254 len 8192 26/30 Write block 332 len 8192 data: 22 19 27/30 Read block 569 len 4096 28/30 Write block 482 len 36864 data: 10 10 11 12 11 11 13 15 12 29/30 Read block 407 len 20480 30/30 Read block 484 len 40960 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:33.155311925Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 T T T 2 040-059 174 174 174 287 287 287 T T T 3 060-079 174 174 174 287 287 287 F F F 4 080-099 173 173 173 285 285 285 T T T 5 100-119 174 174 174 286 286 286 T T T 6 120-139 169 169 169 278 278 278 F F F 7 140-159 174 174 174 287 287 287 F F F 8 160-179 173 173 173 285 285 285 F F F 9 180-199 173 173 173 285 285 285 F F F 10 200-219 170 170 170 280 280 280 F F F 11 220-239 173 173 173 285 285 285 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 167 167 167 273 273 273 T T T 16 320-339 174 174 174 287 287 287 T T T 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 173 173 173 285 285 285 F F F 22 440-459 173 173 173 285 285 285 F F F 23 460-479 173 173 173 285 285 285 F F F 24 480-499 174 174 174 287 287 287 T T T 25 500-519 174 174 174 287 287 287 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 173 173 173 285 285 285 F F F 29 580-599 174 174 174 286 286 286 F F F Max gen: 174, Max flush: 287 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 175 175 175 288 288 288 F F F 3 060-079 174 174 174 287 287 287 F F F 4 080-099 175 175 175 288 288 288 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 174 174 174 287 287 287 F F F 8 160-179 173 173 173 285 285 285 F F F 9 180-199 173 173 173 285 285 285 F F F 10 200-219 170 170 170 280 280 280 F F F 11 220-239 173 173 173 285 285 285 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 169 169 169 278 278 278 F F F 15 300-319 175 175 175 288 288 288 F F F 16 320-339 175 175 175 288 288 288 F F F 17 340-359 166 166 166 272 272 272 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 173 173 173 285 285 285 F F F 22 440-459 173 173 173 285 285 285 F F F 23 460-479 173 173 173 285 285 285 F F F 24 480-499 175 175 175 288 288 288 F F F 25 500-519 174 174 174 287 287 287 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 173 173 173 285 285 285 F F F 29 580-599 174 174 174 286 286 286 F F F Max gen: 175, Max flush: 288 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.404927516Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.449031127Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.449777258Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.44983137Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.449849398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.449864495Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} {"msg":"Crucible 89e8fe42-1bc2-48aa-9515-ffc17c4fd80c has session id: 462d704d-f019-4860-9cb7-01ba82bba4a1","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.450438742Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Upstairs opts: Upstairs UUID: 89e8fe42-1bc2-48aa-9515-ffc17c4fd80c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.450482635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Crucible stats registered with UUID: 89e8fe42-1bc2-48aa-9515-ffc17c4fd80c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.450505932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:37.450526579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.450990941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451045764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} {"msg":"89e8fe42-1bc2-48aa-9515-ffc17c4fd80c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45107084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451084248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451105375Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451118743Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45113711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451150798Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451164666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451177344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451190882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451204241Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52025)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451526134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33557)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451576136Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38914)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.451593524Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45216989Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.452208104Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45223803Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.452771382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.452808647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.452837983Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453373565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453480249Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[0]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.4535431Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[0]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453571816Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453587383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[1]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453601132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[1]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45361509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453629237Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[2]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 285, 285, 280, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453642695Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[2]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 173, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453656084Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453669871Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.4536829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453696008Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453727883Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453754889Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"":"downstairs","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453780475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453804602Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"0","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453831088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"1","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453856824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"client":"2","":"downstairs","session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453881611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453905307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"89e8fe42-1bc2-48aa-9515-ffc17c4fd80c is now active with session: 462d704d-f019-4860-9cb7-01ba82bba4a1"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453934563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.45395543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886,"session_id":"462d704d-f019-4860-9cb7-01ba82bba4a1"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:37.453968178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 581 len 8192 data: 13 14 02/30 Write block 291 len 36864 data: 18 14 14 17 19 15 13 11 11 03/30 Read block 118 len 40960 04/30 Read block 124 len 16384 05/30 Read block 171 len 24576 06/30 Read block 169 len 36864 07/30 Read block 587 len 8192 08/30 Read block 69 len 28672 09/30 Read block 257 len 4096 10/30 Read block 115 len 12288 11/30 Flush 12/30 Write block 199 len 8192 data: 15 16 13/30 Write block 484 len 8192 data: 12 13 14/30 Read block 18 len 16384 15/30 Read block 29 len 16384 16/30 Write block 196 len 24576 data: 16 15 16 16 17 15 17/30 Read block 130 len 8192 18/30 Write block 595 len 16384 data: 6 9 8 6 19/30 Write block 161 len 8192 data: 16 14 20/30 Flush 21/30 Write block 433 len 4096 data: 13 22/30 Flush 23/30 Write block 435 len 28672 data: 16 16 13 12 11 11 14 24/30 Write block 578 len 20480 data: 14 12 11 14 15 25/30 Read block 482 len 32768 26/30 Write block 339 len 40960 data: 19 16 13 14 13 15 16 19 17 16 27/30 Write block 493 len 20480 data: 7 7 7 9 12 28/30 Write block 458 len 12288 data: 16 19 19 29/30 Read block 218 len 12288 30/30 Write block 161 len 28672 data: 17 15 12 12 14 13 10 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:37.471586104Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 175 175 175 288 288 288 F F F 3 060-079 174 174 174 287 287 287 F F F 4 080-099 175 175 175 288 288 288 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 174 174 174 287 287 287 F F F 8 160-179 176 176 176 290 290 290 T T T 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 173 173 173 285 285 285 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 175 175 175 288 288 288 F F F 16 320-339 175 175 175 288 288 288 T T T 17 340-359 166 166 166 272 272 272 T T T 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 176 176 176 291 291 291 T T T 22 440-459 173 173 173 285 285 285 T T T 23 460-479 173 173 173 285 285 285 T T T 24 480-499 176 176 176 290 290 290 T T T 25 500-519 174 174 174 287 287 287 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 173 173 173 285 285 285 T T T 29 580-599 176 176 176 290 290 290 T T T Max gen: 176, Max flush: 291 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 175 175 175 288 288 288 F F F 3 060-079 174 174 174 287 287 287 F F F 4 080-099 175 175 175 288 288 288 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 174 174 174 287 287 287 F F F 8 160-179 177 177 177 292 292 292 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 173 173 173 285 285 285 F F F 12 240-259 168 168 168 275 275 275 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 175 175 175 288 288 288 F F F 16 320-339 177 177 177 292 292 292 F F F 17 340-359 177 177 177 292 292 292 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 167 167 167 273 273 273 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 177 177 177 292 292 292 F F F 23 460-479 177 177 177 292 292 292 F F F 24 480-499 177 177 177 292 292 292 F F F 25 500-519 174 174 174 287 287 287 F F F 26 520-539 171 171 171 283 283 283 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 177 177 177 292 292 292 F F F 29 580-599 177 177 177 292 292 292 F F F Max gen: 177, Max flush: 292 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.735396059Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.780025323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.780637584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.780685927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.780703605Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.780721572Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} {"msg":"Crucible af1c7837-ff82-4e42-9ab2-30b94e8c04d2 has session id: e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781178465Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Upstairs opts: Upstairs UUID: af1c7837-ff82-4e42-9ab2-30b94e8c04d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781224739Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Crucible stats registered with UUID: af1c7837-ff82-4e42-9ab2-30b94e8c04d2","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781251905Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:41.781273142Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781782408Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781810413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} {"msg":"af1c7837-ff82-4e42-9ab2-30b94e8c04d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781868505Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781886142Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781910689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781924807Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781941714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.781965451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782029972Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.78211022Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782139825Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782167492Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"ds_connection connected from Ok(127.0.0.1:37521)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782195647Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"ds_connection connected from Ok(127.0.0.1:54471)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782223013Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"ds_connection connected from Ok(127.0.0.1:37919)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782301302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782803869Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.782847273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.783458993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.78348509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.783580206Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784044518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784070764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784085522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[0]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 292, 290, 290, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.78458029Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[0]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 177, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784601007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784615675Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[1]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 292, 290, 290, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784629313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[1]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 177, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784642781Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784656449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[2]R flush_numbers[0..12]: [282, 288, 288, 287, 288, 288, 278, 287, 292, 290, 290, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784669897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[2]R generation[0..12]: [170, 175, 175, 174, 175, 175, 169, 174, 177, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784686755Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784701073Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784713881Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784726799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784739467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:17:41.784751945Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"":"downstairs","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784767993Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784779691Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"0","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784792509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"1","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784805168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"client":"2","":"downstairs","session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784817856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784829674Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"af1c7837-ff82-4e42-9ab2-30b94e8c04d2 is now active with session: e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784842132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784853641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894,"session_id":"e239e2ae-2d2a-40ae-8b52-f2ec1bdaaa74"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:41.784865349Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 8192 data: 16 14 02/30 Read block 132 len 16384 03/30 Write block 514 len 28672 data: 11 10 10 12 13 14 13 04/30 Read block 581 len 32768 05/30 Read block 499 len 24576 06/30 Write block 475 len 36864 data: 16 16 16 14 12 11 11 11 11 07/30 Read block 406 len 12288 08/30 Read block 335 len 28672 09/30 Write block 231 len 40960 data: 12 17 13 15 17 17 14 16 17 19 10/30 Write block 60 len 16384 data: 10 13 17 22 11/30 Read block 71 len 16384 12/30 Flush 13/30 Write block 342 len 8192 data: 15 14 14/30 Read block 503 len 12288 15/30 Read block 314 len 40960 16/30 Flush 17/30 Write block 530 len 36864 data: 12 9 10 11 12 15 16 18 16 18/30 Write block 50 len 8192 data: 10 13 19/30 Read block 334 len 12288 20/30 Write block 50 len 12288 data: 11 14 13 21/30 Write block 476 len 36864 data: 17 17 15 13 12 12 12 12 13 22/30 Write block 404 len 20480 data: 22 20 17 16 16 23/30 Read block 315 len 20480 24/30 Flush 25/30 Write block 147 len 12288 data: 19 23 25 26/30 Read block 272 len 4096 27/30 Read block 102 len 8192 28/30 Write block 311 len 4096 data: 11 29/30 Write block 233 len 4096 data: 14 30/30 Read block 520 len 32768 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 5 4 6 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:41.803401911Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 178 178 178 295 295 295 F F F 3 060-079 178 178 178 293 293 293 F F F 4 080-099 175 175 175 288 288 288 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 174 174 174 287 287 287 T T T 8 160-179 177 177 177 292 292 292 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 178 178 178 293 293 293 T T T 12 240-259 178 178 178 293 293 293 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 175 175 175 288 288 288 T T T 16 320-339 177 177 177 292 292 292 F F F 17 340-359 178 178 178 294 294 294 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 178 178 178 295 295 295 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 177 177 177 292 292 292 F F F 23 460-479 178 178 178 295 295 295 F F F 24 480-499 178 178 178 295 295 295 F F F 25 500-519 178 178 178 293 293 293 F F F 26 520-539 178 178 178 295 295 295 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 177 177 177 292 292 292 F F F 29 580-599 177 177 177 292 292 292 F F F Max gen: 178, Max flush: 295 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 178 178 178 295 295 295 F F F 3 060-079 178 178 178 293 293 293 F F F 4 080-099 175 175 175 288 288 288 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 179 179 179 296 296 296 F F F 8 160-179 177 177 177 292 292 292 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 179 179 179 296 296 296 F F F 12 240-259 178 178 178 293 293 293 F F F 13 260-279 169 169 169 278 278 278 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 179 179 179 296 296 296 F F F 16 320-339 177 177 177 292 292 292 F F F 17 340-359 178 178 178 294 294 294 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 178 178 178 295 295 295 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 177 177 177 292 292 292 F F F 23 460-479 178 178 178 295 295 295 F F F 24 480-499 178 178 178 295 295 295 F F F 25 500-519 178 178 178 293 293 293 F F F 26 520-539 178 178 178 295 295 295 F F F 27 540-559 173 173 173 285 285 285 F F F 28 560-579 177 177 177 292 292 292 F F F 29 580-599 177 177 177 292 292 292 F F F Max gen: 179, Max flush: 296 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.031900915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:46.076168793Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.076536699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.108401092Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.108977118Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.109016872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.10903357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.109055657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} {"msg":"Crucible b04ab1cb-09a6-4dbc-8edd-b016cdf76dcb has session id: 1cc13dbf-60fc-46c8-9752-7a368fc7bc3c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.109597728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Upstairs opts: Upstairs UUID: b04ab1cb-09a6-4dbc-8edd-b016cdf76dcb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.10964612Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Crucible stats registered with UUID: b04ab1cb-09a6-4dbc-8edd-b016cdf76dcb","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.109665648Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:46.109681355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110086876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110112973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} {"msg":"b04ab1cb-09a6-4dbc-8edd-b016cdf76dcb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110146368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110162645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110188232Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110210758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110241904Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110274619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110307644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.11033921Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110368095Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110396591Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36861)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110424437Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63853)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110453313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44224)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.110557078Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.111202134Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.111238229Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.111267664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.11185106Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.111891684Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.111981641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112421717Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112558117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[0]R flush_numbers[0..12]: [282, 288, 295, 293, 288, 288, 278, 296, 292, 290, 290, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112618988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[0]R generation[0..12]: [170, 175, 178, 178, 175, 175, 169, 179, 177, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112650563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112666451Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[1]R flush_numbers[0..12]: [282, 288, 295, 293, 288, 288, 278, 296, 292, 290, 290, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112680849Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[1]R generation[0..12]: [170, 175, 178, 178, 175, 175, 169, 179, 177, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112695007Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112709145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[2]R flush_numbers[0..12]: [282, 288, 295, 293, 288, 288, 278, 296, 292, 290, 290, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112722543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[2]R generation[0..12]: [170, 175, 178, 178, 175, 175, 169, 179, 177, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112735791Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112749429Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112761927Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112774875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112787643Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.11280696Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"":"downstairs","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112819959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112831537Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"0","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112847635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"1","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112860923Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"client":"2","":"downstairs","session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112873761Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112885669Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"b04ab1cb-09a6-4dbc-8edd-b016cdf76dcb is now active with session: 1cc13dbf-60fc-46c8-9752-7a368fc7bc3c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112898017Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:46.112909536Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902,"session_id":"1cc13dbf-60fc-46c8-9752-7a368fc7bc3c"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-14T16:17:46.112921474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 550 len 32768 data: 13 14 12 15 13 15 18 17 02/30 Write block 410 len 16384 data: 16 13 15 13 03/30 Read block 271 len 4096 04/30 Write block 539 len 32768 data: 15 14 14 14 15 13 13 12 05/30 Read block 66 len 24576 06/30 Read block 128 len 32768 07/30 Write block 56 len 16384 data: 11 13 12 9 08/30 Read block 105 len 4096 09/30 Write block 289 len 4096 data: 19 10/30 Write block 259 len 36864 data: 11 9 8 8 10 13 15 15 13 11/30 Read block 340 len 36864 12/30 Read block 569 len 20480 13/30 Write block 83 len 8192 data: 9 6 14/30 Read block 576 len 24576 15/30 Write block 546 len 12288 data: 13 14 17 16/30 Flush 17/30 Write block 285 len 32768 data: 18 16 15 15 20 19 19 15 18/30 Write block 221 len 28672 data: 12 14 14 12 11 13 12 19/30 Write block 338 len 28672 data: 19 20 17 14 16 15 16 20/30 Write block 297 len 32768 data: 14 12 12 10 10 10 10 11 21/30 Read block 497 len 36864 22/30 Read block 62 len 24576 23/30 Read block 296 len 40960 24/30 Read block 186 len 32768 25/30 Write block 576 len 36864 data: 19 16 15 13 12 15 16 10 14 26/30 Write block 551 len 16384 data: 15 13 16 14 27/30 Write block 563 len 40960 data: 10 8 7 9 11 15 14 14 16 17 28/30 Write block 528 len 32768 data: 10 9 13 10 11 12 13 16 29/30 Write block 223 len 4096 data: 15 30/30 Read block 597 len 8192 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:46.132161794Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 180 180 180 297 297 297 F F F 3 060-079 178 178 178 293 293 293 F F F 4 080-099 180 180 180 297 297 297 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 179 179 179 296 296 296 F F F 8 160-179 177 177 177 292 292 292 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 179 179 179 296 296 296 T T T 12 240-259 180 180 180 297 297 297 F F F 13 260-279 180 180 180 297 297 297 F F F 14 280-299 180 180 180 297 297 297 T T T 15 300-319 179 179 179 296 296 296 T T T 16 320-339 177 177 177 292 292 292 T T T 17 340-359 178 178 178 294 294 294 T T T 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 180 180 180 297 297 297 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 177 177 177 292 292 292 F F F 23 460-479 178 178 178 295 295 295 F F F 24 480-499 178 178 178 295 295 295 F F F 25 500-519 178 178 178 293 293 293 F F F 26 520-539 180 180 180 297 297 297 T T T 27 540-559 180 180 180 297 297 297 T T T 28 560-579 177 177 177 292 292 292 T T T 29 580-599 177 177 177 292 292 292 T T T Max gen: 180, Max flush: 297 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 180 180 180 297 297 297 F F F 3 060-079 178 178 178 293 293 293 F F F 4 080-099 180 180 180 297 297 297 F F F 5 100-119 175 175 175 288 288 288 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 179 179 179 296 296 296 F F F 8 160-179 177 177 177 292 292 292 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 181 181 181 298 298 298 F F F 12 240-259 180 180 180 297 297 297 F F F 13 260-279 180 180 180 297 297 297 F F F 14 280-299 181 181 181 298 298 298 F F F 15 300-319 181 181 181 298 298 298 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 181 181 181 298 298 298 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 166 166 166 271 271 271 F F F 20 400-419 180 180 180 297 297 297 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 177 177 177 292 292 292 F F F 23 460-479 178 178 178 295 295 295 F F F 24 480-499 178 178 178 295 295 295 F F F 25 500-519 178 178 178 293 293 293 F F F 26 520-539 181 181 181 298 298 298 F F F 27 540-559 181 181 181 298 298 298 F F F 28 560-579 181 181 181 298 298 298 F F F 29 580-599 181 181 181 298 298 298 F F F Max gen: 181, Max flush: 298 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.427537965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.471798113Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.472430021Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.472481773Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.472499861Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.472518018Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} {"msg":"Crucible 03185e7b-361c-4e35-8a4d-d940f2002df3 has session id: 2024c601-5c78-4eb9-89c4-9b1d0d6b6a95","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473085286Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Upstairs opts: Upstairs UUID: 03185e7b-361c-4e35-8a4d-d940f2002df3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473129579Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Crucible stats registered with UUID: 03185e7b-361c-4e35-8a4d-d940f2002df3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473154246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:50.473180382Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473665941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473697357Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} {"msg":"03185e7b-361c-4e35-8a4d-d940f2002df3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473751829Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473785154Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473832027Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473862203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473889449Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473917205Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473943621Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473968727Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.473994133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.47401942Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"ds_connection connected from Ok(127.0.0.1:59683)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.474046816Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"ds_connection connected from Ok(127.0.0.1:32771)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.474072062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"ds_connection connected from Ok(127.0.0.1:36480)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.474096409Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.474668125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.474826172Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.475269388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.475434824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.475920603Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.47601181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.476084609Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.47656285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[0]R flush_numbers[0..12]: [282, 288, 297, 293, 297, 288, 278, 296, 292, 290, 290, 298]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477048619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[0]R generation[0..12]: [170, 175, 180, 178, 180, 175, 169, 179, 177, 176, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477072425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477090393Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[1]R flush_numbers[0..12]: [282, 288, 297, 293, 297, 288, 278, 296, 292, 290, 290, 298]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.47710715Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[1]R generation[0..12]: [170, 175, 180, 178, 180, 175, 169, 179, 177, 176, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477123808Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477142095Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[2]R flush_numbers[0..12]: [282, 288, 297, 293, 297, 288, 278, 296, 292, 290, 290, 298]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477171211Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[2]R generation[0..12]: [170, 175, 180, 178, 180, 175, 169, 179, 177, 176, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477204456Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477243751Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477263597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477286164Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-14T16:17:50.47731461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477345676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"":"downstairs","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477377051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477396568Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"0","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477418055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"1","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477446071Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"client":"2","":"downstairs","session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477477926Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477501033Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"03185e7b-361c-4e35-8a4d-d940f2002df3 is now active with session: 2024c601-5c78-4eb9-89c4-9b1d0d6b6a95","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.47752207Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477549146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910,"session_id":"2024c601-5c78-4eb9-89c4-9b1d0d6b6a95"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:50.477576772Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 458 len 36864 data: 17 20 20 15 17 14 14 16 16 02/30 Read block 318 len 40960 03/30 Write block 396 len 32768 data: 16 19 20 22 20 18 17 21 04/30 Write block 166 len 40960 data: 14 11 9 12 10 11 11 16 14 10 05/30 Read block 505 len 16384 06/30 Read block 187 len 36864 07/30 Read block 287 len 28672 08/30 Write block 491 len 8192 data: 11 10 09/30 Flush 10/30 Write block 496 len 20480 data: 10 13 9 14 15 11/30 Read block 478 len 4096 12/30 Write block 95 len 40960 data: 10 13 13 14 17 18 17 14 12 15 13/30 Read block 127 len 20480 14/30 Read block 254 len 20480 15/30 Write block 13 len 4096 data: 7 16/30 Read block 530 len 28672 17/30 Write block 231 len 20480 data: 13 18 15 16 18 18/30 Read block 132 len 16384 19/30 Write block 584 len 28672 data: 15 15 14 12 13 11 10 20/30 Read block 366 len 32768 21/30 Read block 191 len 16384 22/30 Flush 23/30 Write block 477 len 20480 data: 18 16 14 13 13 24/30 Read block 278 len 16384 25/30 Write block 176 len 4096 data: 11 26/30 Write block 77 len 24576 data: 13 12 13 12 9 7 27/30 Read block 14 len 36864 28/30 Read block 520 len 40960 29/30 Write block 313 len 28672 data: 16 14 15 12 13 10 8 30/30 Read block 331 len 32768 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:50.498605191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 180 180 180 297 297 297 F F F 3 060-079 178 178 178 293 293 293 T T T 4 080-099 182 182 182 300 300 300 T T T 5 100-119 182 182 182 300 300 300 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 179 179 179 296 296 296 F F F 8 160-179 182 182 182 299 299 299 T T T 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 182 182 182 300 300 300 F F F 12 240-259 180 180 180 297 297 297 F F F 13 260-279 180 180 180 297 297 297 F F F 14 280-299 181 181 181 298 298 298 F F F 15 300-319 181 181 181 298 298 298 T T T 16 320-339 181 181 181 298 298 298 F F F 17 340-359 181 181 181 298 298 298 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 182 182 182 299 299 299 F F F 20 400-419 182 182 182 299 299 299 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 182 182 182 299 299 299 F F F 23 460-479 182 182 182 299 299 299 T T T 24 480-499 182 182 182 300 300 300 T T T 25 500-519 182 182 182 300 300 300 F F F 26 520-539 181 181 181 298 298 298 F F F 27 540-559 181 181 181 298 298 298 F F F 28 560-579 181 181 181 298 298 298 F F F 29 580-599 182 182 182 300 300 300 F F F Max gen: 182, Max flush: 300 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 180 180 180 297 297 297 F F F 3 060-079 183 183 183 301 301 301 F F F 4 080-099 183 183 183 301 301 301 F F F 5 100-119 182 182 182 300 300 300 F F F 6 120-139 169 169 169 278 278 278 F F F 7 140-159 179 179 179 296 296 296 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 182 182 182 300 300 300 F F F 12 240-259 180 180 180 297 297 297 F F F 13 260-279 180 180 180 297 297 297 F F F 14 280-299 181 181 181 298 298 298 F F F 15 300-319 183 183 183 301 301 301 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 181 181 181 298 298 298 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 182 182 182 299 299 299 F F F 20 400-419 182 182 182 299 299 299 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 182 182 182 299 299 299 F F F 23 460-479 183 183 183 301 301 301 F F F 24 480-499 183 183 183 301 301 301 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 181 181 181 298 298 298 F F F 27 540-559 181 181 181 298 298 298 F F F 28 560-579 181 181 181 298 298 298 F F F 29 580-599 182 182 182 300 300 300 F F F Max gen: 183, Max flush: 301 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.746153704Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:54.788815294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.789218915Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.820878788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.821463673Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.821498028Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.821512826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.821527543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} {"msg":"Crucible 5da556dd-702f-4ce7-a1ef-8cca59d6367a has session id: aa6459a0-4f33-4349-b0bb-3e5ae64ca273","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822135845Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Upstairs opts: Upstairs UUID: 5da556dd-702f-4ce7-a1ef-8cca59d6367a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822179518Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Crucible stats registered with UUID: 5da556dd-702f-4ce7-a1ef-8cca59d6367a","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822203575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:54.822224312Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822617875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82265246Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} {"msg":"5da556dd-702f-4ce7-a1ef-8cca59d6367a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82271828Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822747906Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82278697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822817876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822847121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822875447Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822903143Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82292846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822958575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.822984081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"ds_connection connected from Ok(127.0.0.1:48515)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.823010388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"ds_connection connected from Ok(127.0.0.1:60164)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.823134479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"ds_connection connected from Ok(127.0.0.1:35153)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.823161735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82368176Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.823711426Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.823840337Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.824372239Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.824428701Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.824460817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825029234Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825062349Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[0]R flush_numbers[0..12]: [300, 288, 297, 301, 301, 300, 278, 296, 301, 290, 290, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825090185Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[0]R generation[0..12]: [182, 175, 180, 183, 183, 182, 169, 179, 183, 176, 176, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825111152Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825130519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[1]R flush_numbers[0..12]: [300, 288, 297, 301, 301, 300, 278, 296, 301, 290, 290, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825145037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[1]R generation[0..12]: [182, 175, 180, 183, 183, 182, 169, 179, 183, 176, 176, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825158605Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825172203Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[2]R flush_numbers[0..12]: [300, 288, 297, 301, 301, 300, 278, 296, 301, 290, 290, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825185531Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[2]R generation[0..12]: [182, 175, 180, 183, 183, 182, 169, 179, 183, 176, 176, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825198899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825212497Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825225015Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825241893Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.82526436Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825294565Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"":"downstairs","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825312082Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825324191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"0","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825337389Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"1","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825352976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"client":"2","":"downstairs","session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825366285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825378383Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"5da556dd-702f-4ce7-a1ef-8cca59d6367a is now active with session: aa6459a0-4f33-4349-b0bb-3e5ae64ca273","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825390941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825402519Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918,"session_id":"aa6459a0-4f33-4349-b0bb-3e5ae64ca273"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:54.825414428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 568 len 8192 data: 16 15 02/30 Read block 193 len 24576 03/30 Read block 514 len 24576 04/30 Write block 403 len 28672 data: 22 23 21 18 17 17 15 05/30 Flush 06/30 Write block 357 len 24576 data: 11 11 10 10 13 12 07/30 Write block 220 len 20480 data: 9 13 15 16 13 08/30 Write block 454 len 32768 data: 17 17 15 15 18 21 21 16 09/30 Write block 490 len 36864 data: 13 12 11 8 8 8 11 14 10 10/30 Write block 139 len 36864 data: 10 10 12 14 14 17 19 16 20 11/30 Read block 594 len 12288 12/30 Write block 288 len 8192 data: 16 21 13/30 Read block 4 len 40960 14/30 Write block 468 len 4096 data: 14 15/30 Write block 125 len 40960 data: 7 6 7 9 12 13 12 13 15 15 16/30 Write block 574 len 16384 data: 19 18 20 17 17/30 Read block 467 len 36864 18/30 Read block 343 len 4096 19/30 Write block 371 len 32768 data: 13 10 9 10 12 11 12 12 20/30 Write block 375 len 36864 data: 13 12 13 13 11 11 12 12 12 21/30 Write block 223 len 12288 data: 17 14 12 22/30 Flush 23/30 Write block 526 len 36864 data: 10 10 11 10 14 11 12 13 14 24/30 Read block 209 len 4096 25/30 Read block 397 len 16384 26/30 Flush 27/30 Write block 52 len 32768 data: 14 13 11 10 12 14 13 10 28/30 Write block 273 len 40960 data: 12 15 16 16 16 14 13 14 16 15 29/30 Read block 565 len 40960 30/30 Write block 546 len 40960 data: 14 15 18 16 14 16 14 17 15 16 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:54.844352131Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 180 180 180 297 297 297 T T T 3 060-079 183 183 183 301 301 301 F F F 4 080-099 183 183 183 301 301 301 F F F 5 100-119 182 182 182 300 300 300 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 184 184 184 303 303 303 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 184 184 184 303 303 303 F F F 12 240-259 180 180 180 297 297 297 F F F 13 260-279 180 180 180 297 297 297 T T T 14 280-299 184 184 184 303 303 303 T T T 15 300-319 183 183 183 301 301 301 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 184 184 184 303 303 303 F F F 18 360-379 184 184 184 303 303 303 F F F 19 380-399 184 184 184 303 303 303 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 184 184 184 304 304 304 F F F 27 540-559 181 181 181 298 298 298 T T T 28 560-579 184 184 184 303 303 303 F F F 29 580-599 182 182 182 300 300 300 F F F Max gen: 184, Max flush: 304 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 183 183 183 301 301 301 F F F 4 080-099 183 183 183 301 301 301 F F F 5 100-119 182 182 182 300 300 300 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 184 184 184 303 303 303 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 184 184 184 303 303 303 F F F 12 240-259 180 180 180 297 297 297 F F F 13 260-279 185 185 185 305 305 305 F F F 14 280-299 185 185 185 305 305 305 F F F 15 300-319 183 183 183 301 301 301 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 184 184 184 303 303 303 F F F 18 360-379 184 184 184 303 303 303 F F F 19 380-399 184 184 184 303 303 303 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 184 184 184 304 304 304 F F F 27 540-559 185 185 185 305 305 305 F F F 28 560-579 184 184 184 303 303 303 F F F 29 580-599 182 182 182 300 300 300 F F F Max gen: 185, Max flush: 305 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.109667397Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.153912508Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.154681846Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.154729899Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.154749326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.154767274Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} {"msg":"Crucible abdc5127-5137-4f6a-9c31-928217e0f651 has session id: 7230a394-0b78-4337-ad62-2b3d762be3b7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.155389153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Upstairs opts: Upstairs UUID: abdc5127-5137-4f6a-9c31-928217e0f651, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.155436146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Crucible stats registered with UUID: abdc5127-5137-4f6a-9c31-928217e0f651","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.155455264Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:59.155470861Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.155957831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.155993115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} {"msg":"abdc5127-5137-4f6a-9c31-928217e0f651 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156054706Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156073114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15609775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156126896Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156156551Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156183598Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156211543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15623973Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156266606Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156293682Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61599)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156321538Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46752)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156348994Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56747)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156377619Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.156958175Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.157084276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.157441604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.157581694Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.157616589Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.157977826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158148932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15822564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[0]R flush_numbers[0..12]: [300, 288, 305, 301, 301, 300, 303, 303, 301, 290, 290, 303]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158700311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[0]R generation[0..12]: [182, 175, 185, 183, 183, 182, 184, 184, 183, 176, 176, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158724418Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158744115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[1]R flush_numbers[0..12]: [300, 288, 305, 301, 301, 300, 303, 303, 301, 290, 290, 303]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158761932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[1]R generation[0..12]: [182, 175, 185, 183, 183, 182, 184, 184, 183, 176, 176, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15877907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158796247Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[2]R flush_numbers[0..12]: [300, 288, 305, 301, 301, 300, 303, 303, 301, 290, 290, 303]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158813285Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[2]R generation[0..12]: [182, 175, 185, 183, 183, 182, 184, 184, 183, 176, 176, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158829873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.1588469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158874706Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15891334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158932138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.158957504Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"":"downstairs","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15898707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159014126Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"0","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159038592Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"1","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.15905547Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"client":"2","":"downstairs","session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159071687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159087645Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"abdc5127-5137-4f6a-9c31-928217e0f651 is now active with session: 7230a394-0b78-4337-ad62-2b3d762be3b7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159114851Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159146176Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926,"session_id":"7230a394-0b78-4337-ad62-2b3d762be3b7"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:17:59.159163344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 12288 data: 23 19 23 02/30 Read block 521 len 24576 03/30 Read block 293 len 40960 04/30 Read block 131 len 12288 05/30 Read block 344 len 28672 06/30 Read block 441 len 4096 07/30 Read block 252 len 40960 08/30 Read block 87 len 40960 09/30 Write block 463 len 28672 data: 15 15 17 17 13 15 12 10/30 Flush 11/30 Write block 561 len 28672 data: 11 11 11 9 8 10 12 12/30 Write block 90 len 28672 data: 8 8 8 7 7 11 14 13/30 Read block 237 len 36864 14/30 Read block 545 len 20480 15/30 Read block 176 len 16384 16/30 Write block 148 len 12288 data: 24 26 18 17/30 Flush 18/30 Write block 594 len 4096 data: 7 19/30 Read block 158 len 32768 20/30 Read block 463 len 8192 21/30 Write block 571 len 4096 data: 17 22/30 Write block 235 len 28672 data: 19 18 15 17 18 20 17 23/30 Write block 301 len 28672 data: 11 11 11 12 7 8 10 24/30 Flush 25/30 Write block 231 len 20480 data: 14 19 16 17 20 26/30 Read block 13 len 32768 27/30 Write block 474 len 8192 data: 16 17 28/30 Read block 21 len 16384 29/30 Read block 207 len 16384 30/30 Read block 460 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:17:59.178261374Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 186 186 186 306 306 306 F F F 4 080-099 186 186 186 307 307 307 F F F 5 100-119 182 182 182 300 300 300 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 186 186 186 308 308 308 T T T 12 240-259 186 186 186 308 308 308 F F F 13 260-279 185 185 185 305 305 305 F F F 14 280-299 185 185 185 305 305 305 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 184 184 184 303 303 303 F F F 18 360-379 184 184 184 303 303 303 F F F 19 380-399 184 184 184 303 303 303 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 186 186 186 306 306 306 T T T 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 184 184 184 304 304 304 F F F 27 540-559 185 185 185 305 305 305 F F F 28 560-579 186 186 186 308 308 308 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 186, Max flush: 308 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 186 186 186 306 306 306 F F F 4 080-099 186 186 186 307 307 307 F F F 5 100-119 182 182 182 300 300 300 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 187 187 187 309 309 309 F F F 12 240-259 186 186 186 308 308 308 F F F 13 260-279 185 185 185 305 305 305 F F F 14 280-299 185 185 185 305 305 305 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 184 184 184 303 303 303 F F F 18 360-379 184 184 184 303 303 303 F F F 19 380-399 184 184 184 303 303 303 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 184 184 184 304 304 304 F F F 27 540-559 185 185 185 305 305 305 F F F 28 560-579 186 186 186 308 308 308 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 187, Max flush: 309 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.396380359Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.442460023Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443035689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443086732Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443105719Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443124326Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} {"msg":"Crucible 36f781e6-ba15-4c44-9981-8793a8dbd0f8 has session id: cb22b9c9-5942-4c7f-8d62-615851a1fd3b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443663368Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Upstairs opts: Upstairs UUID: 36f781e6-ba15-4c44-9981-8793a8dbd0f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443705412Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Crucible stats registered with UUID: 36f781e6-ba15-4c44-9981-8793a8dbd0f8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.443724469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:03.443740457Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444215947Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444239834Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} {"msg":"36f781e6-ba15-4c44-9981-8793a8dbd0f8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444296026Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444322872Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444348788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444363856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444378974Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444401051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444443114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.44447227Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444499886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444525862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38930)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444549219Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52677)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444574735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51496)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.444706296Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.445278873Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.445306379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.44577817Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.445949685Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.445975081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446391121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446547168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446634475Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[0]R flush_numbers[0..12]: [300, 288, 305, 306, 307, 300, 303, 307, 301, 290, 290, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446885089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[0]R generation[0..12]: [182, 175, 185, 186, 186, 182, 184, 186, 183, 176, 176, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446904676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446919044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[1]R flush_numbers[0..12]: [300, 288, 305, 306, 307, 300, 303, 307, 301, 290, 290, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446932862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[1]R generation[0..12]: [182, 175, 185, 186, 186, 182, 184, 186, 183, 176, 176, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.44694659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446960448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[2]R flush_numbers[0..12]: [300, 288, 305, 306, 307, 300, 303, 307, 301, 290, 290, 309]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446978305Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[2]R generation[0..12]: [182, 175, 185, 186, 186, 182, 184, 186, 183, 176, 176, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.446992153Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447005851Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.44701839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447031238Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447043886Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447056364Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"":"downstairs","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447076521Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447098248Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"0","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447122584Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"1","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447147121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"client":"2","":"downstairs","session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447170957Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447192984Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"36f781e6-ba15-4c44-9981-8793a8dbd0f8 is now active with session: cb22b9c9-5942-4c7f-8d62-615851a1fd3b","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447216461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447240627Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934,"session_id":"cb22b9c9-5942-4c7f-8d62-615851a1fd3b"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:03.447263294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 32768 data: 11 10 11 14 13 14 14 12 02/30 Write block 225 len 40960 data: 13 14 13 9 12 12 15 20 17 18 03/30 Write block 524 len 36864 data: 8 9 11 11 12 11 15 12 13 04/30 Write block 79 len 40960 data: 14 13 10 8 10 7 8 9 10 10 05/30 Write block 92 len 40960 data: 9 8 8 12 15 14 15 18 19 18 06/30 Write block 404 len 16384 data: 24 22 19 18 07/30 Read block 139 len 8192 08/30 Read block 371 len 28672 09/30 Write block 410 len 32768 data: 17 14 16 14 14 13 13 11 10/30 Write block 351 len 40960 data: 16 14 15 16 15 15 12 12 11 11 11/30 Write block 195 len 12288 data: 14 17 16 12/30 Write block 241 len 24576 data: 18 18 15 12 13 12 13/30 Read block 248 len 32768 14/30 Read block 538 len 8192 15/30 Read block 578 len 24576 16/30 Read block 50 len 20480 17/30 Write block 225 len 8192 data: 14 15 18/30 Read block 435 len 24576 19/30 Read block 214 len 40960 20/30 Write block 556 len 8192 data: 19 18 21/30 Write block 241 len 40960 data: 19 19 16 13 14 13 12 12 14 12 22/30 Read block 222 len 12288 23/30 Read block 450 len 28672 24/30 Read block 375 len 8192 25/30 Write block 408 len 20480 data: 18 16 18 15 17 26/30 Write block 254 len 28672 data: 14 11 12 12 10 12 10 27/30 Write block 563 len 12288 data: 12 10 9 28/30 Write block 6 len 28672 data: 9 11 11 13 14 11 11 29/30 Write block 366 len 12288 data: 16 16 16 30/30 Read block 57 len 36864 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:03.465040067Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 T T T 1 020-039 175 175 175 288 288 288 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 186 186 186 306 306 306 T T T 4 080-099 186 186 186 307 307 307 T T T 5 100-119 182 182 182 300 300 300 T T T 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 176 176 176 290 290 290 T T T 10 200-219 176 176 176 290 290 290 F F F 11 220-239 187 187 187 309 309 309 T T T 12 240-259 186 186 186 308 308 308 T T T 13 260-279 185 185 185 305 305 305 T T T 14 280-299 185 185 185 305 305 305 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 184 184 184 303 303 303 T T T 18 360-379 184 184 184 303 303 303 T T T 19 380-399 184 184 184 303 303 303 F F F 20 400-419 184 184 184 302 302 302 T T T 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 184 184 184 304 304 304 T T T 27 540-559 185 185 185 305 305 305 T T T 28 560-579 186 186 186 308 308 308 T T T 29 580-599 186 186 186 308 308 308 F F F Max gen: 187, Max flush: 309 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 310 310 310 F F F 1 020-039 175 175 175 288 288 288 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 188 188 188 310 310 310 F F F 4 080-099 188 188 188 310 310 310 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 188 188 188 310 310 310 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 188 188 188 310 310 310 F F F 12 240-259 188 188 188 310 310 310 F F F 13 260-279 188 188 188 310 310 310 F F F 14 280-299 185 185 185 305 305 305 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 188 188 188 310 310 310 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 184 184 184 303 303 303 F F F 20 400-419 188 188 188 310 310 310 F F F 21 420-439 177 177 177 292 292 292 F F F 22 440-459 184 184 184 303 303 303 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 188 188 188 310 310 310 F F F 27 540-559 188 188 188 310 310 310 F F F 28 560-579 188 188 188 310 310 310 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 188, Max flush: 310 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.751983324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.795771831Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.796380192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.796422726Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.796437484Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.796451622Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} {"msg":"Crucible 5aa0d5ee-0724-4422-b4ed-0d740d8e422c has session id: 11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797068282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Upstairs opts: Upstairs UUID: 5aa0d5ee-0724-4422-b4ed-0d740d8e422c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797122284Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Crucible stats registered with UUID: 5aa0d5ee-0724-4422-b4ed-0d740d8e422c","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.79715088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:17.797176876Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797666775Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.79770135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} {"msg":"5aa0d5ee-0724-4422-b4ed-0d740d8e422c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.79776765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797785618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797807544Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797826392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797840939Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797910109Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797950403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.79797559Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.797999386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798024313Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:53738)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798051089Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37511)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798100892Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45657)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798235132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798715402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798750417Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.798788821Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.79935179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.799481311Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.799509037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.799845898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800026201Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[0]R flush_numbers[0..12]: [310, 288, 305, 310, 310, 310, 303, 307, 301, 310, 290, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800136056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[0]R generation[0..12]: [188, 175, 185, 188, 188, 188, 184, 186, 183, 188, 176, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800155882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.80017038Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[1]R flush_numbers[0..12]: [310, 288, 305, 310, 310, 310, 303, 307, 301, 310, 290, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800184328Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[1]R generation[0..12]: [188, 175, 185, 188, 188, 188, 184, 186, 183, 188, 176, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800197707Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800211474Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[2]R flush_numbers[0..12]: [310, 288, 305, 310, 310, 310, 303, 307, 301, 310, 290, 310]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800224912Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[2]R generation[0..12]: [188, 175, 185, 188, 188, 188, 184, 186, 183, 188, 176, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.80023827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800252099Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800264657Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800281294Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800294352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800313799Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"":"downstairs","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800326728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800338266Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"0","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800351124Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"1","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800363672Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"client":"2","":"downstairs","session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.80037643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800388509Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"5aa0d5ee-0724-4422-b4ed-0d740d8e422c is now active with session: 11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800401037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800412735Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942,"session_id":"11ee27c5-b483-4d67-b12d-b6c3c4b9b1bd"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:17.800424623Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 24576 data: 15 13 16 13 11 16 02/30 Write block 237 len 24576 data: 16 18 19 21 20 20 03/30 Write block 542 len 12288 data: 15 16 14 04/30 Read block 54 len 32768 05/30 Write block 288 len 12288 data: 17 22 20 06/30 Read block 343 len 12288 07/30 Read block 387 len 40960 08/30 Read block 467 len 24576 09/30 Flush 10/30 Flush 11/30 Write block 81 len 32768 data: 11 9 11 8 9 10 11 11 12/30 Read block 382 len 40960 13/30 Write block 576 len 8192 data: 21 18 14/30 Read block 390 len 28672 15/30 Read block 208 len 16384 16/30 Read block 265 len 32768 17/30 Flush 18/30 Write block 447 len 40960 data: 15 13 12 13 15 13 17 18 18 16 19/30 Write block 421 len 40960 data: 16 18 15 14 15 10 9 12 13 15 20/30 Write block 239 len 32768 data: 20 22 21 21 17 14 15 14 21/30 Write block 33 len 24576 data: 16 16 13 13 13 16 22/30 Read block 206 len 16384 23/30 Flush 24/30 Write block 422 len 32768 data: 19 16 15 16 11 10 13 14 25/30 Write block 432 len 12288 data: 15 14 12 26/30 Write block 237 len 16384 data: 17 19 21 23 27/30 Read block 272 len 28672 28/30 Read block 312 len 40960 29/30 Write block 387 len 40960 data: 12 16 17 20 18 19 17 14 15 17 30/30 Write block 567 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:17.819081738Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 310 310 310 F F F 1 020-039 190 190 190 314 314 314 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 188 188 188 310 310 310 F F F 4 080-099 190 190 190 313 313 313 F F F 5 100-119 190 190 190 311 311 311 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 188 188 188 310 310 310 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 190 190 190 314 314 314 T T T 12 240-259 190 190 190 314 314 314 T T T 13 260-279 188 188 188 310 310 310 F F F 14 280-299 190 190 190 311 311 311 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 188 188 188 310 310 310 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 184 184 184 303 303 303 T T T 20 400-419 188 188 188 310 310 310 F F F 21 420-439 190 190 190 314 314 314 T T T 22 440-459 190 190 190 314 314 314 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 188 188 188 310 310 310 F F F 27 540-559 190 190 190 311 311 311 F F F 28 560-579 190 190 190 313 313 313 T T T 29 580-599 186 186 186 308 308 308 F F F Max gen: 190, Max flush: 314 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 310 310 310 F F F 1 020-039 190 190 190 314 314 314 F F F 2 040-059 185 185 185 305 305 305 F F F 3 060-079 188 188 188 310 310 310 F F F 4 080-099 190 190 190 313 313 313 F F F 5 100-119 190 190 190 311 311 311 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 188 188 188 310 310 310 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 191 191 191 315 315 315 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 188 188 188 310 310 310 F F F 14 280-299 190 190 190 311 311 311 F F F 15 300-319 186 186 186 308 308 308 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 188 188 188 310 310 310 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 188 188 188 310 310 310 F F F 21 420-439 191 191 191 315 315 315 F F F 22 440-459 190 190 190 314 314 314 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 188 188 188 310 310 310 F F F 27 540-559 190 190 190 311 311 311 F F F 28 560-579 191 191 191 315 315 315 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 191, Max flush: 315 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.059415329Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.103554676Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104178635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104228607Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104247564Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104265322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} {"msg":"Crucible 29453eb6-4cbe-4b93-a12b-5aa83d7bfca8 has session id: 1d696002-41cd-4eb2-b785-8565650bdd37","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104855616Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Upstairs opts: Upstairs UUID: 29453eb6-4cbe-4b93-a12b-5aa83d7bfca8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104890681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Crucible stats registered with UUID: 29453eb6-4cbe-4b93-a12b-5aa83d7bfca8","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.104906179Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:22.104923646Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105361922Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105386209Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} {"msg":"29453eb6-4cbe-4b93-a12b-5aa83d7bfca8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.10544988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105467467Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105490494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105504752Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.10551879Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105537037Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105562563Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.10558661Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105607687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105631643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"ds_connection connected from Ok(127.0.0.1:59958)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105695894Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"ds_connection connected from Ok(127.0.0.1:65336)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105732108Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"ds_connection connected from Ok(127.0.0.1:53580)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.105768713Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.106463012Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.106502566Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.10661401Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107107088Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107135554Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"1","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107151632Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107665387Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107690494Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[0]R flush_numbers[0..12]: [310, 314, 305, 310, 313, 311, 303, 307, 301, 310, 290, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107717379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[0]R generation[0..12]: [188, 190, 185, 188, 190, 190, 184, 186, 183, 188, 176, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107731968Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107745875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[1]R flush_numbers[0..12]: [310, 314, 305, 310, 313, 311, 303, 307, 301, 310, 290, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107759344Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[1]R generation[0..12]: [188, 190, 185, 188, 190, 190, 184, 186, 183, 188, 176, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107772611Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107789839Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[2]R flush_numbers[0..12]: [310, 314, 305, 310, 313, 311, 303, 307, 301, 310, 290, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107804077Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[2]R generation[0..12]: [188, 190, 185, 188, 190, 190, 184, 186, 183, 188, 176, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107817635Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107831403Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107844081Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107856959Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107869537Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107881936Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"":"downstairs","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107894734Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107906322Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"0","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.1079192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"1","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107940317Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"client":"2","":"downstairs","session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107954795Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107966933Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"29453eb6-4cbe-4b93-a12b-5aa83d7bfca8 is now active with session: 1d696002-41cd-4eb2-b785-8565650bdd37","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.107979272Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.1079908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950,"session_id":"1d696002-41cd-4eb2-b785-8565650bdd37"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:22.108004388Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 20480 data: 10 9 10 13 17 02/30 Write block 167 len 36864 data: 12 10 13 11 12 12 17 15 11 03/30 Write block 193 len 4096 data: 17 04/30 Write block 295 len 36864 data: 20 16 15 13 13 11 12 12 12 05/30 Read block 585 len 12288 06/30 Write block 401 len 20480 data: 19 18 23 25 23 07/30 Read block 508 len 12288 08/30 Write block 59 len 32768 data: 11 11 14 18 24 20 24 18 09/30 Read block 317 len 16384 10/30 Read block 366 len 16384 11/30 Write block 178 len 20480 data: 11 12 12 9 10 12/30 Write block 347 len 20480 data: 18 17 17 16 17 13/30 Write block 55 len 36864 data: 11 13 15 14 12 12 15 19 25 14/30 Read block 330 len 36864 15/30 Flush 16/30 Flush 17/30 Write block 155 len 16384 data: 13 13 14 12 18/30 Read block 483 len 20480 19/30 Read block 434 len 16384 20/30 Read block 341 len 24576 21/30 Read block 549 len 16384 22/30 Read block 392 len 28672 23/30 Write block 143 len 16384 data: 15 18 20 17 24/30 Write block 23 len 8192 data: 6 7 25/30 Write block 278 len 24576 data: 15 14 15 17 16 14 26/30 Read block 84 len 12288 27/30 Write block 480 len 8192 data: 14 14 28/30 Read block 134 len 20480 29/30 Write block 54 len 8192 data: 12 12 30/30 Write block 45 len 8192 data: 12 13 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:22.126168854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 310 310 310 F F F 1 020-039 190 190 190 314 314 314 T T T 2 040-059 192 192 192 316 316 316 T T T 3 060-079 192 192 192 316 316 316 F F F 4 080-099 190 190 190 313 313 313 F F F 5 100-119 190 190 190 311 311 311 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 186 186 186 307 307 307 T T T 8 160-179 192 192 192 316 316 316 F F F 9 180-199 192 192 192 316 316 316 F F F 10 200-219 192 192 192 316 316 316 F F F 11 220-239 191 191 191 315 315 315 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 188 188 188 310 310 310 T T T 14 280-299 192 192 192 316 316 316 T T T 15 300-319 192 192 192 316 316 316 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 192 192 192 316 316 316 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 192 192 192 316 316 316 F F F 21 420-439 191 191 191 315 315 315 F F F 22 440-459 190 190 190 314 314 314 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 184 184 184 303 303 303 T T T 25 500-519 182 182 182 300 300 300 F F F 26 520-539 188 188 188 310 310 310 F F F 27 540-559 190 190 190 311 311 311 F F F 28 560-579 191 191 191 315 315 315 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 192, Max flush: 316 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 310 310 310 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 193 193 193 317 317 317 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 190 190 190 313 313 313 F F F 5 100-119 190 190 190 311 311 311 F F F 6 120-139 184 184 184 303 303 303 F F F 7 140-159 193 193 193 317 317 317 F F F 8 160-179 192 192 192 316 316 316 F F F 9 180-199 192 192 192 316 316 316 F F F 10 200-219 192 192 192 316 316 316 F F F 11 220-239 191 191 191 315 315 315 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 193 193 193 317 317 317 F F F 14 280-299 193 193 193 317 317 317 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 192 192 192 316 316 316 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 192 192 192 316 316 316 F F F 21 420-439 191 191 191 315 315 315 F F F 22 440-459 190 190 190 314 314 314 F F F 23 460-479 187 187 187 309 309 309 F F F 24 480-499 193 193 193 317 317 317 F F F 25 500-519 182 182 182 300 300 300 F F F 26 520-539 188 188 188 310 310 310 F F F 27 540-559 190 190 190 311 311 311 F F F 28 560-579 191 191 191 315 315 315 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 193, Max flush: 317 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.367670276Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:26.410486114Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.410898854Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.442677469Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.443302618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.443344062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.44336133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.443378977Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} {"msg":"Crucible 3afb9e56-0de1-428e-8d75-4fb305e4e0ae has session id: 05d68047-6760-4234-9e13-d50febffaee7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444017334Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Upstairs opts: Upstairs UUID: 3afb9e56-0de1-428e-8d75-4fb305e4e0ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444088824Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Crucible stats registered with UUID: 3afb9e56-0de1-428e-8d75-4fb305e4e0ae","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.44411583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:26.444132968Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444623366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444649273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} {"msg":"3afb9e56-0de1-428e-8d75-4fb305e4e0ae active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444709954Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444727141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444749358Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444763146Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444782133Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444796151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444819448Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444844254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444865461Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444907255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63606)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444924062Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43839)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.444947759Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40616)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.445153119Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.445475162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.445634699Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.446005135Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.446033641Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.446193488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.44665618Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.446701534Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.44672766Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[0]R flush_numbers[0..12]: [310, 317, 317, 316, 313, 311, 303, 317, 316, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447152988Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[0]R generation[0..12]: [188, 193, 193, 192, 190, 190, 184, 193, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447174355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447189093Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[1]R flush_numbers[0..12]: [310, 317, 317, 316, 313, 311, 303, 317, 316, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447202981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[1]R generation[0..12]: [188, 193, 193, 192, 190, 190, 184, 193, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447216339Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447229937Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[2]R flush_numbers[0..12]: [310, 317, 317, 316, 313, 311, 303, 317, 316, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447243325Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[2]R generation[0..12]: [188, 193, 193, 192, 190, 190, 184, 193, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447256583Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447270191Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447282659Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447295417Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447307966Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"All extents match","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-14T16:18:26.447330192Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"":"downstairs","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447354679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447377575Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"0","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447403122Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"1","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447428398Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"client":"2","":"downstairs","session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447454324Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447478361Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"3afb9e56-0de1-428e-8d75-4fb305e4e0ae is now active with session: 05d68047-6760-4234-9e13-d50febffaee7","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447503207Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447526044Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958,"session_id":"05d68047-6760-4234-9e13-d50febffaee7"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:26.447543681Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 32768 data: 12 15 11 15 16 15 15 19 02/30 Write block 177 len 28672 data: 10 12 13 13 10 11 13 03/30 Write block 549 len 28672 data: 17 15 17 15 18 16 17 04/30 Read block 220 len 28672 05/30 Read block 139 len 24576 06/30 Write block 106 len 32768 data: 12 17 16 14 12 13 13 14 07/30 Write block 2 len 16384 data: 8 9 10 9 08/30 Read block 560 len 20480 09/30 Write block 479 len 8192 data: 15 15 10/30 Read block 425 len 20480 11/30 Write block 575 len 16384 data: 19 22 19 16 12/30 Write block 218 len 36864 data: 8 8 10 14 16 18 15 15 16 13/30 Read block 27 len 28672 14/30 Read block 303 len 8192 15/30 Write block 94 len 36864 data: 9 13 16 15 16 19 20 19 16 16/30 Write block 118 len 16384 data: 8 7 9 12 17/30 Read block 378 len 16384 18/30 Flush 19/30 Write block 435 len 36864 data: 17 17 14 13 12 12 15 14 17 20/30 Read block 462 len 36864 21/30 Flush 22/30 Write block 111 len 28672 data: 14 14 15 12 10 9 7 23/30 Read block 2 len 8192 24/30 Write block 463 len 24576 data: 16 16 18 18 14 16 25/30 Read block 435 len 24576 26/30 Write block 436 len 8192 data: 18 15 27/30 Write block 532 len 20480 data: 14 14 15 17 17 28/30 Read block 336 len 24576 29/30 Read block 560 len 36864 30/30 Read block 497 len 12288 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:26.467776187Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 193 193 193 317 317 317 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 194 194 194 318 318 318 T T T 6 120-139 194 194 194 318 318 318 F F F 7 140-159 193 193 193 317 317 317 F F F 8 160-179 194 194 194 318 318 318 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 318 318 318 F F F 11 220-239 194 194 194 318 318 318 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 193 193 193 317 317 317 F F F 14 280-299 193 193 193 317 317 317 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 192 192 192 316 316 316 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 192 192 192 316 316 316 F F F 21 420-439 194 194 194 319 319 319 T T T 22 440-459 194 194 194 319 319 319 F F F 23 460-479 194 194 194 318 318 318 T T T 24 480-499 194 194 194 318 318 318 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 188 188 188 310 310 310 T T T 27 540-559 194 194 194 318 318 318 F F F 28 560-579 194 194 194 318 318 318 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 194, Max flush: 319 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 193 193 193 317 317 317 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 195 195 195 320 320 320 F F F 6 120-139 194 194 194 318 318 318 F F F 7 140-159 193 193 193 317 317 317 F F F 8 160-179 194 194 194 318 318 318 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 318 318 318 F F F 11 220-239 194 194 194 318 318 318 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 193 193 193 317 317 317 F F F 14 280-299 193 193 193 317 317 317 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 181 181 181 298 298 298 F F F 17 340-359 192 192 192 316 316 316 F F F 18 360-379 188 188 188 310 310 310 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 192 192 192 316 316 316 F F F 21 420-439 195 195 195 320 320 320 F F F 22 440-459 194 194 194 319 319 319 F F F 23 460-479 195 195 195 320 320 320 F F F 24 480-499 194 194 194 318 318 318 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 195 195 195 320 320 320 F F F 27 540-559 194 194 194 318 318 318 F F F 28 560-579 194 194 194 318 318 318 F F F 29 580-599 186 186 186 308 308 308 F F F Max gen: 195, Max flush: 320 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.735029251Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.778643613Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.779576427Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.779621961Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.779639128Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.779656785Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} {"msg":"Crucible 0c9982d7-15ce-4eb7-a5da-3a9f61560024 has session id: a8a46c08-8446-498b-8dda-93463ad31746","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780275046Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Upstairs opts: Upstairs UUID: 0c9982d7-15ce-4eb7-a5da-3a9f61560024, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780318639Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Crucible stats registered with UUID: 0c9982d7-15ce-4eb7-a5da-3a9f61560024","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780335347Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:30.780348405Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780829225Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780854141Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} {"msg":"0c9982d7-15ce-4eb7-a5da-3a9f61560024 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780914653Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.78093215Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780955067Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780969365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.780983932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.78100416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781031656Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781055392Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781081908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781106754Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"ds_connection connected from Ok(127.0.0.1:33626)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781132601Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:30.781159747Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:30.781186953Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:30.781220438Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"ds_connection connected from Ok(127.0.0.1:52649)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.78141125Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781440446Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781854466Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.781981987Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.782412664Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.782538386Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.782936168Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:30.783037164Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.782247488Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"ds_connection connected from Ok(127.0.0.1:43707)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.782477355Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.783582714Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784144522Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[0]R flush_numbers[0..12]: [318, 317, 317, 316, 318, 320, 318, 317, 318, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784718859Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[0]R generation[0..12]: [194, 193, 193, 192, 194, 195, 194, 193, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784744485Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784760213Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[1]R flush_numbers[0..12]: [318, 317, 317, 316, 318, 320, 318, 317, 318, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784774631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[1]R generation[0..12]: [194, 193, 193, 192, 194, 195, 194, 193, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784788158Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784801976Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[2]R flush_numbers[0..12]: [318, 317, 317, 316, 318, 320, 318, 317, 318, 318, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784815725Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[2]R generation[0..12]: [194, 193, 193, 192, 194, 195, 194, 193, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784829323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.78484289Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784855679Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784868677Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784881255Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784894683Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"":"downstairs","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784907471Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.78491901Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"0","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784934667Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"1","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784948055Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"client":"2","":"downstairs","session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784961273Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"Wait for a query_work_queue command to finish before sending IO a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.78498265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"0c9982d7-15ce-4eb7-a5da-3a9f61560024 is now active with session: a8a46c08-8446-498b-8dda-93463ad31746","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.784995318Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.785006907Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966,"session_id":"a8a46c08-8446-498b-8dda-93463ad31746"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:40.785018875Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 4096 data: 21 02/30 Flush 03/30 Write block 263 len 40960 data: 11 14 16 16 14 11 12 11 11 14 04/30 Read block 419 len 28672 05/30 Flush 06/30 Write block 275 len 36864 data: 17 17 17 16 15 16 18 17 15 07/30 Flush 08/30 Write block 435 len 32768 data: 18 19 16 14 13 13 16 15 09/30 Write block 361 len 40960 data: 14 13 12 12 13 17 17 17 16 16 10/30 Write block 233 len 28672 data: 18 19 21 19 18 20 22 11/30 Read block 275 len 24576 12/30 Read block 562 len 28672 13/30 Flush 14/30 Write block 575 len 24576 data: 20 23 20 17 14 13 15/30 Read block 248 len 4096 16/30 Read block 321 len 36864 17/30 Write block 41 len 4096 data: 14 18/30 Write block 276 len 40960 data: 18 18 17 16 17 19 18 16 16 19 19/30 Read block 37 len 32768 20/30 Read block 129 len 24576 21/30 Write block 332 len 40960 data: 23 20 19 18 17 17 20 22 18 15 22/30 Write block 100 len 8192 data: 21 20 23/30 Read block 498 len 40960 24/30 Read block 326 len 16384 25/30 Read block 382 len 28672 26/30 Read block 109 len 28672 27/30 Write block 190 len 28672 data: 17 19 17 18 14 15 18 28/30 Write block 166 len 24576 data: 15 13 11 14 12 13 29/30 Read block 433 len 4096 30/30 Write block 419 len 40960 data: 16 16 17 20 17 16 17 12 11 14 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:40.805894687Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 193 193 193 317 317 317 T T T 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 195 195 195 320 320 320 T T T 6 120-139 194 194 194 318 318 318 F F F 7 140-159 193 193 193 317 317 317 F F F 8 160-179 194 194 194 318 318 318 T T T 9 180-199 194 194 194 318 318 318 T T T 10 200-219 194 194 194 318 318 318 F F F 11 220-239 196 196 196 324 324 324 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 196 196 196 323 323 323 T T T 14 280-299 196 196 196 323 323 323 T T T 15 300-319 192 192 192 316 316 316 F F F 16 320-339 196 196 196 321 321 321 T T T 17 340-359 192 192 192 316 316 316 T T T 18 360-379 196 196 196 324 324 324 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 192 192 192 316 316 316 T T T 21 420-439 196 196 196 324 324 324 T T T 22 440-459 196 196 196 324 324 324 F F F 23 460-479 195 195 195 320 320 320 F F F 24 480-499 194 194 194 318 318 318 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 195 195 195 320 320 320 F F F 27 540-559 194 194 194 318 318 318 F F F 28 560-579 194 194 194 318 318 318 T T T 29 580-599 186 186 186 308 308 308 T T T Max gen: 196, Max flush: 324 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 197 197 197 325 325 325 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 197 197 197 325 325 325 F F F 6 120-139 194 194 194 318 318 318 F F F 7 140-159 193 193 193 317 317 317 F F F 8 160-179 197 197 197 325 325 325 F F F 9 180-199 197 197 197 325 325 325 F F F 10 200-219 194 194 194 318 318 318 F F F 11 220-239 196 196 196 324 324 324 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 197 197 197 325 325 325 F F F 14 280-299 197 197 197 325 325 325 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 197 197 197 325 325 325 F F F 17 340-359 197 197 197 325 325 325 F F F 18 360-379 196 196 196 324 324 324 F F F 19 380-399 191 191 191 315 315 315 F F F 20 400-419 197 197 197 325 325 325 F F F 21 420-439 197 197 197 325 325 325 F F F 22 440-459 196 196 196 324 324 324 F F F 23 460-479 195 195 195 320 320 320 F F F 24 480-499 194 194 194 318 318 318 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 195 195 195 320 320 320 F F F 27 540-559 194 194 194 318 318 318 F F F 28 560-579 197 197 197 325 325 325 F F F 29 580-599 197 197 197 325 325 325 F F F Max gen: 197, Max flush: 325 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.081995373Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.126704056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.127322256Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.127373758Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.127392736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.127410413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} {"msg":"Crucible 3c0b662f-3d7e-49e7-8bf3-26f216d62caa has session id: a6ba55ac-9655-48c7-9f07-6e161e7d5195","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128001197Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Upstairs opts: Upstairs UUID: 3c0b662f-3d7e-49e7-8bf3-26f216d62caa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128044151Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Crucible stats registered with UUID: 3c0b662f-3d7e-49e7-8bf3-26f216d62caa","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128067477Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:45.128087685Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128483877Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128518102Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} {"msg":"3c0b662f-3d7e-49e7-8bf3-26f216d62caa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128549597Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128564315Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128585832Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.12860053Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128614788Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128627736Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128641524Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.12866779Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128686178Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128706115Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"ds_connection connected from Ok(127.0.0.1:51656)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128850254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"ds_connection connected from Ok(127.0.0.1:37290)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.128981215Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"ds_connection connected from Ok(127.0.0.1:59535)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.129035117Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.12949636Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.129526625Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.130134767Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.130295513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.130329898Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.13066483Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.130823396Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.130897256Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[0]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 317, 325, 325, 318, 324]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131388254Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[0]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 193, 197, 197, 194, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131412941Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131428079Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[1]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 317, 325, 325, 318, 324]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131441897Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[1]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 193, 197, 197, 194, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131455515Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131469182Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[2]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 317, 325, 325, 318, 324]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131482631Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[2]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 193, 197, 197, 194, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131496169Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131513496Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131526574Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131539842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.13155243Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131565128Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"":"downstairs","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131583866Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131595614Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"0","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131608712Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"1","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.13162162Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"client":"2","":"downstairs","session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131634558Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131646637Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"3c0b662f-3d7e-49e7-8bf3-26f216d62caa is now active with session: a6ba55ac-9655-48c7-9f07-6e161e7d5195","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131659145Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131670763Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974,"session_id":"a6ba55ac-9655-48c7-9f07-6e161e7d5195"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.131682622Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 589 len 8192 data: 12 11 02/30 Write block 469 len 8192 data: 13 12 03/30 Write block 143 len 8192 data: 16 19 04/30 Read block 142 len 12288 05/30 Read block 132 len 8192 06/30 Read block 152 len 4096 07/30 Write block 228 len 12288 data: 10 13 13 08/30 Read block 324 len 32768 09/30 Read block 212 len 32768 10/30 Read block 355 len 36864 11/30 Read block 285 len 20480 12/30 Read block 4 len 24576 13/30 Read block 398 len 8192 14/30 Write block 274 len 4096 data: 16 15/30 Read block 390 len 20480 16/30 Write block 484 len 24576 data: 14 14 12 12 14 16 17/30 Read block 375 len 28672 18/30 Write block 551 len 32768 data: 18 16 19 17 18 20 19 16 19/30 Write block 397 len 4096 data: 20 20/30 Read block 121 len 32768 21/30 Write block 163 len 8192 data: 13 13 22/30 Write block 144 len 12288 data: 20 21 18 23/30 Flush 24/30 Write block 458 len 36864 data: 19 22 22 17 18 17 17 19 19 25/30 Read block 21 len 32768 26/30 Read block 107 len 28672 27/30 Read block 426 len 16384 28/30 Read block 105 len 20480 29/30 Write block 412 len 4096 data: 18 30/30 Write block 133 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:45.148849183Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 197 197 197 325 325 325 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 197 197 197 325 325 325 F F F 6 120-139 194 194 194 318 318 318 T T T 7 140-159 198 198 198 326 326 326 F F F 8 160-179 198 198 198 326 326 326 F F F 9 180-199 197 197 197 325 325 325 F F F 10 200-219 194 194 194 318 318 318 F F F 11 220-239 198 198 198 326 326 326 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 198 198 198 326 326 326 F F F 14 280-299 197 197 197 325 325 325 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 197 197 197 325 325 325 F F F 17 340-359 197 197 197 325 325 325 F F F 18 360-379 196 196 196 324 324 324 F F F 19 380-399 198 198 198 326 326 326 F F F 20 400-419 197 197 197 325 325 325 T T T 21 420-439 197 197 197 325 325 325 F F F 22 440-459 196 196 196 324 324 324 T T T 23 460-479 198 198 198 326 326 326 T T T 24 480-499 198 198 198 326 326 326 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 195 195 195 320 320 320 F F F 27 540-559 198 198 198 326 326 326 F F F 28 560-579 197 197 197 325 325 325 F F F 29 580-599 198 198 198 326 326 326 F F F Max gen: 198, Max flush: 326 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 318 318 318 F F F 1 020-039 193 193 193 317 317 317 F F F 2 040-059 197 197 197 325 325 325 F F F 3 060-079 192 192 192 316 316 316 F F F 4 080-099 194 194 194 318 318 318 F F F 5 100-119 197 197 197 325 325 325 F F F 6 120-139 199 199 199 327 327 327 F F F 7 140-159 198 198 198 326 326 326 F F F 8 160-179 198 198 198 326 326 326 F F F 9 180-199 197 197 197 325 325 325 F F F 10 200-219 194 194 194 318 318 318 F F F 11 220-239 198 198 198 326 326 326 F F F 12 240-259 191 191 191 315 315 315 F F F 13 260-279 198 198 198 326 326 326 F F F 14 280-299 197 197 197 325 325 325 F F F 15 300-319 192 192 192 316 316 316 F F F 16 320-339 197 197 197 325 325 325 F F F 17 340-359 197 197 197 325 325 325 F F F 18 360-379 196 196 196 324 324 324 F F F 19 380-399 198 198 198 326 326 326 F F F 20 400-419 199 199 199 327 327 327 F F F 21 420-439 197 197 197 325 325 325 F F F 22 440-459 199 199 199 327 327 327 F F F 23 460-479 199 199 199 327 327 327 F F F 24 480-499 198 198 198 326 326 326 F F F 25 500-519 194 194 194 318 318 318 F F F 26 520-539 195 195 195 320 320 320 F F F 27 540-559 198 198 198 326 326 326 F F F 28 560-579 197 197 197 325 325 325 F F F 29 580-599 198 198 198 326 326 326 F F F Max gen: 199, Max flush: 327 8:24 Test duration Test completed real 8:23.756181174 user 49.046922745 sys 18.836005920 trap 0.071978111 tflt 0.080587381 dflt 0.153352538 kflt 0.006480647 lock 13:28:31.904052932 slp 40:58.075993179 lat 5.799140890 stop 0.060418869