Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27242333 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.021153476 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.021255886 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1186 1187 1188 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.213425885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.283948348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.285804386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.285956313Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.285981767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.286000023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} {"msg":"Crucible f9d539e3-d4d3-4de0-8164-10c29e297886 has session id: 6848f25a-36b0-4021-969e-6805b18fc5d4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.287100065Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Upstairs opts: Upstairs UUID: f9d539e3-d4d3-4de0-8164-10c29e297886, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.287220169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Crucible stats registered with UUID: f9d539e3-d4d3-4de0-8164-10c29e297886","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.28725995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:32.287287824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288273891Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288363782Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} {"msg":"f9d539e3-d4d3-4de0-8164-10c29e297886 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288391596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288406862Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288449553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288475867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288503421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288529296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.28855521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288581414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288605419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288628574Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61849)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288652159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55720)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288675594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51018)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.288881809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.290712463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.290740177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.290766001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.291598241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.291634793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.291676174Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292390079Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292415634Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292568911Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292587356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292601484Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292614411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292627318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292640725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292653632Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292672228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292700642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292724817Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292750691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292767408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292780175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"":"downstairs","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.29280055Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292826515Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"0","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292852129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"1","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292878053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"client":"2","":"downstairs","session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292903188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292926853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"f9d539e3-d4d3-4de0-8164-10c29e297886 is now active with session: 6848f25a-36b0-4021-969e-6805b18fc5d4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292949968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:32.292970524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190,"session_id":"6848f25a-36b0-4021-969e-6805b18fc5d4"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2025-01-23T17:37:32.292995998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:36.385997942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1190} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.418709636Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.464564662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.465300343Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.465344973Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.46536071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.465376307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} {"msg":"Crucible b9cfc8ac-279c-4216-83ea-d37c2f567580 has session id: f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.465962919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Upstairs opts: Upstairs UUID: b9cfc8ac-279c-4216-83ea-d37c2f567580, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.46600665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Crucible stats registered with UUID: b9cfc8ac-279c-4216-83ea-d37c2f567580","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466030435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:36.466076915Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466544984Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466574677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} {"msg":"b9cfc8ac-279c-4216-83ea-d37c2f567580 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466630385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466657069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466683483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.46669886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466713387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466726524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466740461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466753348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466766856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466780602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65174)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.46688593Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40643)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466922412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57492)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.466947966Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.467648894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.467687546Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.468249024Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.468388014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.468877368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.468944684Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.46923659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.469772704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.47030237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470321095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470335632Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470349739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470363116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470376783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.47039292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470406657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470420264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470432841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470445508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470457806Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470470323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"":"downstairs","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470482771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470502096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"0","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470515013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"1","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470527681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"client":"2","":"downstairs","session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470540458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470552205Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"b9cfc8ac-279c-4216-83ea-d37c2f567580 is now active with session: f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470564583Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.47057607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192,"session_id":"f9c101c5-41b9-41d5-97c2-e94fdfa5c1cd"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:36.470588168Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 565 len 20480 data: 2 2 2 2 2 02/30 Write block 115 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 60 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Flush 05/30 Write block 21 len 32768 data: 2 2 2 2 2 2 2 2 06/30 Flush 07/30 Write block 448 len 16384 data: 2 2 2 2 08/30 Read block 184 len 16384 09/30 Read block 315 len 32768 10/30 Read block 580 len 16384 11/30 Read block 124 len 24576 12/30 Read block 527 len 4096 13/30 Read block 456 len 20480 14/30 Write block 84 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 404 len 28672 16/30 Read block 253 len 8192 17/30 Flush 18/30 Write block 453 len 24576 data: 2 2 2 2 2 2 19/30 Read block 128 len 4096 20/30 Read block 196 len 12288 21/30 Read block 231 len 40960 22/30 Read block 404 len 40960 23/30 Read block 62 len 32768 24/30 Write block 328 len 20480 data: 2 2 2 2 2 25/30 Read block 142 len 20480 26/30 Write block 484 len 4096 data: 2 27/30 Write block 537 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Read block 568 len 4096 29/30 Write block 34 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Write block 500 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:36.491587249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1192} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 T T T 2 040-059 1 1 1 1 1 1 T T T 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 4 4 4 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 4 4 4 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 T T T 27 540-559 1 1 1 1 1 1 T T T 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 3 3 3 5 5 5 F F F 2 040-059 3 3 3 5 5 5 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 4 4 4 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 3 3 3 5 5 5 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.751754142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:40.794397104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.794885078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.826262341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.826964019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.8270073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.827030715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.827049201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} {"msg":"Crucible 2dd8e7c2-e68f-4036-89b4-c065b8ce721c has session id: 3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.827696491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Upstairs opts: Upstairs UUID: 2dd8e7c2-e68f-4036-89b4-c065b8ce721c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.827750689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Crucible stats registered with UUID: 2dd8e7c2-e68f-4036-89b4-c065b8ce721c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.827778663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:40.827805027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828242722Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828269306Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} {"msg":"2dd8e7c2-e68f-4036-89b4-c065b8ce721c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828327524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828353028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828377133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.82839124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828405497Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828418554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828465164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828500246Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.82852826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828553575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"ds_connection connected from Ok(127.0.0.1:61749)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.82857683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"ds_connection connected from Ok(127.0.0.1:62307)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828604384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"ds_connection connected from Ok(127.0.0.1:44332)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.828817528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.8292677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.829445732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.829485433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.829945603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830029815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830130113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830591253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830616018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[0]R flush_numbers[0..12]: [1, 5, 5, 2, 4, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830634274Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[0]R generation[0..12]: [1, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830649141Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830663418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[1]R flush_numbers[0..12]: [1, 5, 5, 2, 4, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830676635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[1]R generation[0..12]: [1, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830689402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830702689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[2]R flush_numbers[0..12]: [1, 5, 5, 2, 4, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830715826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[2]R generation[0..12]: [1, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830729014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830742541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830754998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830767535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830779883Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830798648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"":"downstairs","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830811616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830825613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"0","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.83083889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"1","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830851867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"client":"2","":"downstairs","session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830864864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:37:40.830876742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"2dd8e7c2-e68f-4036-89b4-c065b8ce721c is now active with session: 3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830891148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830902806Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200,"session_id":"3dcbdbb3-f17b-4fc9-afc0-1b83b6fc2466"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:40.830914633Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 271 len 8192 data: 2 2 02/30 Write block 175 len 16384 data: 2 2 2 2 03/30 Read block 386 len 4096 04/30 Write block 9 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 166 len 16384 06/30 Read block 223 len 28672 07/30 Write block 253 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Write block 140 len 16384 data: 2 2 2 2 09/30 Read block 538 len 4096 10/30 Read block 441 len 36864 11/30 Write block 543 len 32768 data: 3 3 2 2 2 2 2 2 12/30 Write block 36 len 24576 data: 3 3 3 3 3 3 13/30 Read block 30 len 36864 14/30 Write block 180 len 36864 data: 2 2 2 2 2 2 2 2 2 15/30 Read block 201 len 4096 16/30 Write block 500 len 36864 data: 3 3 3 3 2 2 2 2 2 17/30 Read block 355 len 16384 18/30 Read block 366 len 32768 19/30 Read block 389 len 36864 20/30 Write block 573 len 8192 data: 2 2 21/30 Read block 515 len 28672 22/30 Read block 313 len 32768 23/30 Read block 91 len 12288 24/30 Read block 292 len 36864 25/30 Read block 77 len 20480 26/30 Flush 27/30 Write block 587 len 8192 data: 2 2 28/30 Write block 544 len 4096 data: 4 29/30 Write block 548 len 32768 data: 3 3 3 2 2 2 2 2 30/30 Write block 16 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:40.850802575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1200} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 T T T 1 020-039 4 4 4 6 6 6 T T T 2 040-059 4 4 4 6 6 6 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 4 4 4 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 6 6 6 F F F 13 260-279 4 4 4 6 6 6 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 4 4 4 6 6 6 T T T 28 560-579 4 4 4 6 6 6 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 4, Max flush: 6 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 4 4 4 6 6 6 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 4 4 4 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 6 6 6 F F F 13 260-279 4 4 4 6 6 6 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.119698647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:45.169030501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.169688218Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.202401912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203108969Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203144221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203158628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203172615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} {"msg":"Crucible cab79269-1c8d-4c9a-9f59-6b4fd7ff13e1 has session id: f40c3051-13df-40fb-894e-691b4299168d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203788701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Upstairs opts: Upstairs UUID: cab79269-1c8d-4c9a-9f59-6b4fd7ff13e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203835481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Crucible stats registered with UUID: cab79269-1c8d-4c9a-9f59-6b4fd7ff13e1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.203859116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:45.203879872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204316237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204342991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} {"msg":"cab79269-1c8d-4c9a-9f59-6b4fd7ff13e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204411307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204433912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204456887Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204476302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204502377Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204529061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204558105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204585519Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204606804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204620741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40193)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204712621Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36200)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204752542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41978)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.204782076Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.205464508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.205492162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.205506809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.206271744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.206579197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.206600492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.20693528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207048155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 6, 2, 4, 2, 2, 6, 6, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207082828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[0]R generation[0..12]: [5, 5, 4, 2, 2, 2, 2, 4, 4, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207097585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207111462Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 6, 2, 4, 2, 2, 6, 6, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207124819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[1]R generation[0..12]: [5, 5, 4, 2, 2, 2, 2, 4, 4, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207137836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207151223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 6, 2, 4, 2, 2, 6, 6, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.20716429Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[2]R generation[0..12]: [5, 5, 4, 2, 2, 2, 2, 4, 4, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207177057Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207190585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207207561Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207221368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207241573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207263069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"":"downstairs","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207293662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207317957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"0","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207339902Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"1","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207353739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"client":"2","":"downstairs","session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207368146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207389591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"cab79269-1c8d-4c9a-9f59-6b4fd7ff13e1 is now active with session: f40c3051-13df-40fb-894e-691b4299168d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207413636Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207432512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208,"session_id":"f40c3051-13df-40fb-894e-691b4299168d"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:45.207444909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 465 len 4096 data: 2 04/30 Read block 558 len 8192 05/30 Read block 426 len 24576 06/30 Read block 322 len 20480 07/30 Read block 427 len 40960 08/30 Read block 220 len 36864 09/30 Read block 449 len 36864 10/30 Write block 114 len 32768 data: 2 3 3 3 3 3 3 3 11/30 Flush 12/30 Write block 255 len 20480 data: 3 3 3 3 3 13/30 Read block 560 len 4096 14/30 Read block 65 len 4096 15/30 Write block 206 len 36864 data: 2 2 2 2 2 2 2 2 2 16/30 Write block 343 len 16384 data: 2 2 2 2 17/30 Flush 18/30 Write block 82 len 16384 data: 2 2 3 3 19/30 Flush 20/30 Write block 46 len 28672 data: 2 2 2 2 2 2 2 21/30 Read block 31 len 8192 22/30 Read block 212 len 16384 23/30 Write block 279 len 8192 data: 2 2 24/30 Write block 139 len 36864 data: 2 3 3 3 3 2 2 2 2 25/30 Read block 136 len 36864 26/30 Read block 153 len 4096 27/30 Write block 159 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 340 len 4096 data: 2 29/30 Write block 398 len 16384 data: 2 2 2 2 30/30 Read block 44 len 40960 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 10 11 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:45.227497095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1208} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 4 4 4 6 6 6 T T T 3 060-079 2 2 2 2 2 2 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 9 9 9 F F F 6 120-139 6 6 6 9 9 9 T T T 7 140-159 4 4 4 6 6 6 T T T 8 160-179 4 4 4 6 6 6 T T T 9 180-199 4 4 4 6 6 6 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 4 4 4 6 6 6 T T T 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 6 6 6 10 10 10 T T T 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 6, Max flush: 11 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 9 9 9 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 7 7 7 12 12 12 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 12 12 12 F F F 20 400-419 7 7 7 12 12 12 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.517571639Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.560643369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.561322712Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.561366552Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.561382199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.561397906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} {"msg":"Crucible dfa1efd9-4991-4d13-b9dd-32bd46b4dfaf has session id: c8020b92-7f26-4b56-90dd-6367cc636bd1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562029829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Upstairs opts: Upstairs UUID: dfa1efd9-4991-4d13-b9dd-32bd46b4dfaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562079418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Crucible stats registered with UUID: dfa1efd9-4991-4d13-b9dd-32bd46b4dfaf","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562111901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:49.562135466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562462795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56248688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} {"msg":"dfa1efd9-4991-4d13-b9dd-32bd46b4dfaf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56253425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562552356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562576461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562590897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562615582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562639977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562692325Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562719929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562740765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56276544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46777)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562790864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35027)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562818408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59277)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.562843243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.563458809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.563485933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56414839Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.564178274Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.564767436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.564831852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.564904306Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565399269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 12, 2, 11, 9, 12, 12, 12, 6, 10, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565862409Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[0]R generation[0..12]: [5, 5, 7, 2, 6, 6, 7, 7, 7, 4, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565885244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56590058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 12, 2, 11, 9, 12, 12, 12, 6, 10, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565916007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[1]R generation[0..12]: [5, 5, 7, 2, 6, 6, 7, 7, 7, 4, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565929824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.56594732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 12, 2, 11, 9, 12, 12, 12, 6, 10, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565962027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[2]R generation[0..12]: [5, 5, 7, 2, 6, 6, 7, 7, 7, 4, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.565985582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566000109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566020084Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566034961Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566050348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566065725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"":"downstairs","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566081731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566096248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"0","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566112655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"1","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566128361Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"client":"2","":"downstairs","session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566144108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566158614Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"dfa1efd9-4991-4d13-b9dd-32bd46b4dfaf is now active with session: c8020b92-7f26-4b56-90dd-6367cc636bd1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566173961Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566191507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216,"session_id":"c8020b92-7f26-4b56-90dd-6367cc636bd1"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:49.566206714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 4096 data: 2 02/30 Write block 310 len 28672 data: 2 2 2 2 2 2 2 03/30 Write block 101 len 12288 data: 2 2 2 04/30 Write block 238 len 20480 data: 2 2 2 2 2 05/30 Read block 308 len 20480 06/30 Read block 260 len 40960 07/30 Read block 355 len 20480 08/30 Write block 592 len 8192 data: 2 2 09/30 Write block 13 len 20480 data: 3 3 3 3 3 10/30 Write block 281 len 20480 data: 2 2 2 2 2 11/30 Write block 134 len 4096 data: 2 12/30 Write block 283 len 20480 data: 3 3 3 2 2 13/30 Write block 56 len 32768 data: 2 2 2 2 3 3 3 3 14/30 Write block 462 len 28672 data: 2 2 2 3 2 2 2 15/30 Read block 293 len 32768 16/30 Read block 352 len 32768 17/30 Read block 469 len 36864 18/30 Write block 401 len 12288 data: 3 2 2 19/30 Read block 40 len 12288 20/30 Write block 525 len 36864 data: 2 2 2 2 2 2 2 2 2 21/30 Read block 25 len 40960 22/30 Write block 187 len 36864 data: 3 3 2 2 2 2 2 2 3 23/30 Write block 480 len 24576 data: 2 2 2 2 3 2 24/30 Read block 510 len 36864 25/30 Read block 354 len 28672 26/30 Write block 520 len 24576 data: 2 2 2 2 2 3 27/30 Read block 383 len 20480 28/30 Write block 97 len 8192 data: 2 2 29/30 Write block 146 len 32768 data: 3 3 2 2 2 2 2 2 30/30 Write block 294 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:49.5846453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1216} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 T T T 1 020-039 5 5 5 7 7 7 F F F 2 040-059 7 7 7 12 12 12 T T T 3 060-079 2 2 2 2 2 2 T T T 4 080-099 6 6 6 11 11 11 T T T 5 100-119 6 6 6 9 9 9 T T T 6 120-139 7 7 7 12 12 12 T T T 7 140-159 7 7 7 12 12 12 T T T 8 160-179 7 7 7 12 12 12 F F F 9 180-199 4 4 4 6 6 6 T T T 10 200-219 6 6 6 10 10 10 F F F 11 220-239 1 1 1 1 1 1 T T T 12 240-259 6 6 6 10 10 10 T T T 13 260-279 7 7 7 12 12 12 F F F 14 280-299 7 7 7 12 12 12 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 3 3 3 5 5 5 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 12 12 12 F F F 20 400-419 7 7 7 12 12 12 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 6 6 6 9 9 9 T T T 24 480-499 3 3 3 5 5 5 T T T 25 500-519 4 4 4 6 6 6 F F F 26 520-539 3 3 3 5 5 5 T T T 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 7 7 7 T T T Max gen: 7, Max flush: 12 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 8 8 8 13 13 13 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 8 8 8 13 13 13 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 8 8 8 13 13 13 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 8 8 8 13 13 13 F F F 15 300-319 8 8 8 13 13 13 F F F 16 320-339 3 3 3 5 5 5 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 12 12 12 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 8, Max flush: 13 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.880851515Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:53.924770031Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.925525067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.957819022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.958768296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.958807117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.958824094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.95884179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} {"msg":"Crucible 51d3b3a1-dd98-41e4-b0bc-2d8cbf2b0eea has session id: 3b4f1bf0-4c19-467d-ae47-d8118c3d308f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.959422474Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Upstairs opts: Upstairs UUID: 51d3b3a1-dd98-41e4-b0bc-2d8cbf2b0eea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.959477792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Crucible stats registered with UUID: 51d3b3a1-dd98-41e4-b0bc-2d8cbf2b0eea","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.959501687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:53.959518094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.95995176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.959980123Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} {"msg":"51d3b3a1-dd98-41e4-b0bc-2d8cbf2b0eea active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960054447Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960079892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960108186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960124782Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960139219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960152316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960166233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96018122Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960205805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96022924Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50043)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96036278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64429)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960400732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55772)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.960426507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961118647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961157219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961294229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961817685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961847249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.961873693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962418385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962444729Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[0]R flush_numbers[0..12]: [13, 7, 13, 13, 13, 13, 13, 13, 12, 13, 10, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962465535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[0]R generation[0..12]: [8, 5, 8, 8, 8, 8, 8, 8, 7, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962480972Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962495249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[1]R flush_numbers[0..12]: [13, 7, 13, 13, 13, 13, 13, 13, 12, 13, 10, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962508656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[1]R generation[0..12]: [8, 5, 8, 8, 8, 8, 8, 8, 7, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962521903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96253542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[2]R flush_numbers[0..12]: [13, 7, 13, 13, 13, 13, 13, 13, 12, 13, 10, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962549547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[2]R generation[0..12]: [8, 5, 8, 8, 8, 8, 8, 8, 7, 8, 6, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962562734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96257982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Max found gen is 9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962593277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Generation requested: 10 >= found:9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962606045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962624921Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962637638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"":"downstairs","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962650995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962662652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"0","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.96267561Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"1","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962689267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"client":"2","":"downstairs","session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962702134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962713771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"51d3b3a1-dd98-41e4-b0bc-2d8cbf2b0eea is now active with session: 3b4f1bf0-4c19-467d-ae47-d8118c3d308f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962726049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962737586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224,"session_id":"3b4f1bf0-4c19-467d-ae47-d8118c3d308f"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:53.962749354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 8192 data: 2 2 02/30 Write block 36 len 28672 data: 4 4 4 4 4 4 2 03/30 Read block 234 len 8192 04/30 Read block 566 len 8192 05/30 Read block 424 len 16384 06/30 Read block 581 len 8192 07/30 Read block 122 len 8192 08/30 Read block 244 len 16384 09/30 Read block 241 len 16384 10/30 Read block 1 len 28672 11/30 Read block 341 len 16384 12/30 Flush 13/30 Write block 498 len 4096 data: 2 14/30 Write block 215 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 233 len 28672 16/30 Read block 229 len 16384 17/30 Write block 451 len 24576 data: 3 2 3 3 3 3 18/30 Write block 27 len 16384 data: 3 3 2 2 19/30 Write block 303 len 16384 data: 2 2 2 2 20/30 Flush 21/30 Write block 373 len 24576 data: 2 2 2 2 2 2 22/30 Write block 483 len 16384 data: 3 4 3 2 23/30 Read block 404 len 40960 24/30 Write block 275 len 16384 data: 2 2 2 2 25/30 Write block 390 len 24576 data: 2 2 2 2 2 2 26/30 Read block 428 len 12288 27/30 Read block 579 len 28672 28/30 Write block 328 len 4096 data: 3 29/30 Read block 535 len 20480 30/30 Write block 331 len 40960 data: 3 3 2 2 2 2 2 2 2 3 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:53.981535744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1224} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 8 8 8 13 13 13 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 7 7 7 12 12 12 T T T 14 280-299 8 8 8 13 13 13 F F F 15 300-319 10 10 10 15 15 15 F F F 16 320-339 3 3 3 5 5 5 T T T 17 340-359 7 7 7 12 12 12 T T T 18 360-379 1 1 1 1 1 1 T T T 19 380-399 7 7 7 12 12 12 T T T 20 400-419 8 8 8 13 13 13 F F F 21 420-439 10 10 10 14 14 14 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 10 10 10 15 15 15 T T T 25 500-519 4 4 4 6 6 6 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 10, Max flush: 15 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 8 8 8 13 13 13 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 11 11 11 16 16 16 F F F 14 280-299 8 8 8 13 13 13 F F F 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 11 11 11 16 16 16 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 10 10 10 14 14 14 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 11 11 11 16 16 16 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 11, Max flush: 16 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.253369599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:58.295728703Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.296263547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.327927188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329037448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329079359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329094955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329111432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} {"msg":"Crucible e27e1767-d145-4b31-b4ab-1728da58762d has session id: 9b64e39a-e976-4909-95b1-567653e82b48","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329792164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Upstairs opts: Upstairs UUID: e27e1767-d145-4b31-b4ab-1728da58762d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329838954Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Crucible stats registered with UUID: e27e1767-d145-4b31-b4ab-1728da58762d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.329862659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:58.329883974Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330433275Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330461869Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} {"msg":"e27e1767-d145-4b31-b4ab-1728da58762d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330517367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330539932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330562777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330629643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330667295Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330684781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330700277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330714194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330728071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.330749407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:63130)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.33110442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:34155)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.331146491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:44843)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.331171795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.331723366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.331758248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.331959735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.332437041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.33248986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.332715481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.33304164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333103117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 14, 13, 13, 13, 13, 13, 12, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333259393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[0]R generation[0..12]: [8, 10, 10, 8, 8, 8, 8, 8, 7, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333278679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333293126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 14, 13, 13, 13, 13, 13, 12, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333311542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[1]R generation[0..12]: [8, 10, 10, 8, 8, 8, 8, 8, 7, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333326159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333339985Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 14, 13, 13, 13, 13, 13, 12, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333353602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[2]R generation[0..12]: [8, 10, 10, 8, 8, 8, 8, 8, 7, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.33336707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333380757Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333393384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Generation requested: 12 >= found:12","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-23T17:37:58.333414219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333426957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333439544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"":"downstairs","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333452201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333463599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"0","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333476266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"1","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333489193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"client":"2","":"downstairs","session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333505929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333518737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"e27e1767-d145-4b31-b4ab-1728da58762d is now active with session: 9b64e39a-e976-4909-95b1-567653e82b48","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333531264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333542901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232,"session_id":"9b64e39a-e976-4909-95b1-567653e82b48"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:37:58.333554929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 261 len 32768 data: 3 2 2 2 2 2 2 2 03/30 Flush 04/30 Write block 308 len 24576 data: 2 2 3 3 3 3 05/30 Flush 06/30 Write block 170 len 12288 data: 2 2 2 07/30 Read block 87 len 20480 08/30 Read block 360 len 16384 09/30 Flush 10/30 Write block 100 len 20480 data: 2 3 3 3 2 11/30 Read block 375 len 20480 12/30 Read block 585 len 4096 13/30 Read block 270 len 36864 14/30 Write block 537 len 28672 data: 3 3 3 3 3 3 4 15/30 Read block 38 len 36864 16/30 Read block 56 len 20480 17/30 Write block 537 len 36864 data: 4 4 4 4 4 4 5 5 3 18/30 Flush 19/30 Write block 417 len 20480 data: 2 2 2 2 2 20/30 Write block 375 len 40960 data: 3 3 3 3 2 2 2 2 2 2 21/30 Write block 276 len 32768 data: 3 3 3 3 3 3 3 4 22/30 Read block 166 len 4096 23/30 Flush 24/30 Write block 470 len 12288 data: 2 2 2 25/30 Read block 55 len 24576 26/30 Read block 58 len 40960 27/30 Write block 521 len 16384 data: 3 3 3 3 28/30 Read block 314 len 4096 29/30 Read block 62 len 40960 30/30 Write block 98 len 12288 data: 3 2 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:37:58.356518404Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1232} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 T T T 5 100-119 12 12 12 21 21 21 T T T 6 120-139 8 8 8 13 13 13 F F F 7 140-159 8 8 8 13 13 13 F F F 8 160-179 12 12 12 20 20 20 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 12 12 12 22 22 22 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 12 12 12 22 22 22 F F F 21 420-439 12 12 12 22 22 22 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 8 8 8 13 13 13 T T T 24 480-499 11 11 11 16 16 16 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 12 12 12 21 21 21 T T T 27 540-559 12 12 12 21 21 21 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 12, Max flush: 22 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 8 8 8 13 13 13 F F F 8 160-179 12 12 12 20 20 20 F F F 9 180-199 8 8 8 13 13 13 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 12 12 12 22 22 22 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 12 12 12 22 22 22 F F F 21 420-439 12 12 12 22 22 22 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 11 11 11 16 16 16 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 13 13 13 23 23 23 F F F 27 540-559 12 12 12 21 21 21 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.62575968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:02.669136714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.669734304Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.700794086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.701574547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.701612659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.701628975Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.701646841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} {"msg":"Crucible 31d7f59a-82bb-469a-9ed1-8cf32aa9decb has session id: 97bba1df-8c1a-480f-8e91-5d1c26c0215e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702308618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Upstairs opts: Upstairs UUID: 31d7f59a-82bb-469a-9ed1-8cf32aa9decb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702350829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Crucible stats registered with UUID: 31d7f59a-82bb-469a-9ed1-8cf32aa9decb","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702368495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:02.702382522Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702824586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702849731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} {"msg":"31d7f59a-82bb-469a-9ed1-8cf32aa9decb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702922685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702949209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.702979063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703005697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703040819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703132959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703154745Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703170521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703195746Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.70322179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35807)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.703247354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50683)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.70327104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47000)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.70350125Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704088992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704128074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704381949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704809136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704874222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.704903606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.7053908Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705503636Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 14, 13, 23, 23, 13, 13, 20, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705562323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[0]R generation[0..12]: [8, 10, 10, 8, 13, 13, 8, 8, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705582069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705596596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 14, 13, 23, 23, 13, 13, 20, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705610073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[1]R generation[0..12]: [8, 10, 10, 8, 13, 13, 8, 8, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.70562323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705637007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 14, 13, 23, 23, 13, 13, 20, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705650184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[2]R generation[0..12]: [8, 10, 10, 8, 13, 13, 8, 8, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705663101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705676398Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705688916Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705707801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705724468Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705737715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"":"downstairs","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705750412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.7057619Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"0","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705775147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"1","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705788374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"client":"2","":"downstairs","session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705801421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705813189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"31d7f59a-82bb-469a-9ed1-8cf32aa9decb is now active with session: 97bba1df-8c1a-480f-8e91-5d1c26c0215e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705826696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705838433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240,"session_id":"97bba1df-8c1a-480f-8e91-5d1c26c0215e"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:02.705850541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 28672 data: 3 3 2 2 3 3 3 02/30 Read block 201 len 28672 03/30 Read block 321 len 12288 04/30 Read block 158 len 36864 05/30 Write block 442 len 36864 data: 2 2 2 2 2 2 3 3 3 06/30 Write block 102 len 16384 data: 4 4 3 2 07/30 Write block 414 len 20480 data: 2 2 2 3 3 08/30 Write block 268 len 24576 data: 3 2 2 3 3 2 09/30 Write block 192 len 12288 data: 3 3 3 10/30 Write block 517 len 4096 data: 2 11/30 Read block 11 len 36864 12/30 Flush 13/30 Write block 434 len 24576 data: 2 2 2 2 2 2 14/30 Write block 509 len 36864 data: 2 2 2 2 2 2 2 2 3 15/30 Write block 156 len 36864 data: 2 2 2 3 3 3 3 3 3 16/30 Read block 145 len 12288 17/30 Write block 273 len 8192 data: 3 2 18/30 Read block 13 len 28672 19/30 Write block 525 len 28672 data: 4 3 3 3 3 3 3 20/30 Read block 394 len 16384 21/30 Read block 231 len 28672 22/30 Read block 282 len 40960 23/30 Flush 24/30 Write block 58 len 20480 data: 3 3 4 4 4 25/30 Read block 174 len 40960 26/30 Read block 528 len 40960 27/30 Read block 89 len 12288 28/30 Read block 485 len 32768 29/30 Read block 568 len 20480 30/30 Read block 174 len 4096 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:02.725509842Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1240} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 14 14 14 T T T 3 060-079 8 8 8 13 13 13 T T T 4 080-099 13 13 13 23 23 23 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 14 14 14 25 25 25 F F F 9 180-199 14 14 14 24 24 24 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 14 14 14 24 24 24 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 11 11 11 16 16 16 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 21 21 21 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 14, Max flush: 25 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 15 15 15 26 26 26 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 8 8 8 13 13 13 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 14 14 14 25 25 25 F F F 9 180-199 14 14 14 24 24 24 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 14 14 14 24 24 24 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 11 11 11 16 16 16 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 21 21 21 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 15, Max flush: 26 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:06.98228201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.02729097Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.028356489Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.028402749Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.028419266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.028434792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} {"msg":"Crucible 53c71a72-767b-4bfa-a5e9-7c785fe4920a has session id: d7e7dd87-83b0-41d7-ad99-a6587575f936","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029049929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Upstairs opts: Upstairs UUID: 53c71a72-767b-4bfa-a5e9-7c785fe4920a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029100738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Crucible stats registered with UUID: 53c71a72-767b-4bfa-a5e9-7c785fe4920a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029122703Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:07.02913702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029630953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029662926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} {"msg":"53c71a72-767b-4bfa-a5e9-7c785fe4920a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029747968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029767964Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029798577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029826351Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029854095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029903494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029937597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029964441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.029989026Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030050353Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"ds_connection connected from Ok(127.0.0.1:50484)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030079186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"ds_connection connected from Ok(127.0.0.1:58494)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030274704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"ds_connection connected from Ok(127.0.0.1:50771)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030302318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030741133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.030791242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.031196164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.031453358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.031567694Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.031980195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032229041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032376249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 26, 26, 23, 24, 13, 25, 25, 24, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032503881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[0]R generation[0..12]: [8, 10, 15, 15, 13, 14, 8, 14, 14, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032522897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032537724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 26, 26, 23, 24, 13, 25, 25, 24, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032551791Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[1]R generation[0..12]: [8, 10, 15, 15, 13, 14, 8, 14, 14, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032565598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032579494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 26, 26, 23, 24, 13, 25, 25, 24, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032592812Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[2]R generation[0..12]: [8, 10, 15, 15, 13, 14, 8, 14, 14, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032609128Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032622955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032643241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032656548Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032669245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032681643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"":"downstairs","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.03269454Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032706297Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"0","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032719264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"1","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032731821Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"client":"2","":"downstairs","session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032744559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032756486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"53c71a72-767b-4bfa-a5e9-7c785fe4920a is now active with session: d7e7dd87-83b0-41d7-ad99-a6587575f936","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032769004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032780601Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248,"session_id":"d7e7dd87-83b0-41d7-ad99-a6587575f936"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:07.032792359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 12288 data: 2 2 2 02/30 Read block 552 len 16384 03/30 Write block 410 len 24576 data: 2 2 2 2 3 3 04/30 Read block 30 len 28672 05/30 Write block 327 len 24576 data: 2 4 3 3 4 4 06/30 Read block 597 len 8192 07/30 Write block 320 len 8192 data: 2 2 08/30 Write block 288 len 8192 data: 2 2 09/30 Write block 129 len 32768 data: 2 2 2 2 2 3 2 2 10/30 Write block 445 len 16384 data: 3 3 3 4 11/30 Flush 12/30 Write block 171 len 32768 data: 3 3 2 2 3 3 3 3 13/30 Flush 14/30 Write block 45 len 24576 data: 2 3 3 3 3 3 15/30 Read block 525 len 8192 16/30 Write block 529 len 4096 data: 4 17/30 Read block 363 len 8192 18/30 Read block 18 len 8192 19/30 Read block 552 len 4096 20/30 Flush 21/30 Write block 438 len 20480 data: 3 3 2 2 3 22/30 Write block 171 len 20480 data: 4 4 3 3 4 23/30 Read block 114 len 12288 24/30 Read block 97 len 40960 25/30 Read block 434 len 40960 26/30 Write block 494 len 12288 data: 2 2 2 27/30 Write block 172 len 36864 data: 5 4 4 5 4 4 4 2 3 28/30 Write block 567 len 40960 data: 3 3 3 2 2 2 3 3 2 2 29/30 Write block 132 len 8192 data: 3 3 30/30 Read block 172 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 2 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 0 5 Done 7 8 10 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:07.052184818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1248} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 16 16 16 27 27 27 T T T 7 140-159 14 14 14 25 25 25 F F F 8 160-179 16 16 16 28 28 28 T T T 9 180-199 14 14 14 24 24 24 T T T 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 16 16 16 27 27 27 F F F 21 420-439 14 14 14 25 25 25 T T T 22 440-459 16 16 16 27 27 27 T T T 23 460-479 13 13 13 23 23 23 F F F 24 480-499 11 11 11 16 16 16 T T T 25 500-519 14 14 14 25 25 25 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 12 12 12 21 21 21 F F F 28 560-579 4 4 4 6 6 6 T T T 29 580-599 8 8 8 13 13 13 F F F Max gen: 16, Max flush: 29 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 17 17 17 30 30 30 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 17 17 17 30 30 30 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 15 15 15 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 16 16 16 27 27 27 F F F 21 420-439 17 17 17 30 30 30 F F F 22 440-459 17 17 17 30 30 30 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 17 17 17 30 30 30 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 12 12 12 21 21 21 F F F 28 560-579 17 17 17 30 30 30 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 17, Max flush: 30 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.311818506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:11.353554365Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.353959547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.384751307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.385401766Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.385443617Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.385460493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.385484138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} {"msg":"Crucible 85bc9fc2-3d7a-46c3-80df-f0aa7ab79576 has session id: 783cd92f-ad3b-4748-bb8b-dd511545a423","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386007365Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Upstairs opts: Upstairs UUID: 85bc9fc2-3d7a-46c3-80df-f0aa7ab79576, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386061563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Crucible stats registered with UUID: 85bc9fc2-3d7a-46c3-80df-f0aa7ab79576","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386082028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:11.386098805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386471124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386501427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} {"msg":"85bc9fc2-3d7a-46c3-80df-f0aa7ab79576 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386544068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386559765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.38658252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386596567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386610914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386632659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386660913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386690217Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386718491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386742685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"ds_connection connected from Ok(127.0.0.1:46583)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.38676823Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"ds_connection connected from Ok(127.0.0.1:59619)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.386794364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"ds_connection connected from Ok(127.0.0.1:52237)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.38695062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.387518577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.387545761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.387560308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388161078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388286181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388342899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388653561Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388805379Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 29, 26, 23, 24, 30, 25, 30, 30, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388940789Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[0]R generation[0..12]: [8, 10, 16, 15, 13, 14, 17, 14, 17, 17, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388964074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388979501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 29, 26, 23, 24, 30, 25, 30, 30, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.388993388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[1]R generation[0..12]: [8, 10, 16, 15, 13, 14, 17, 14, 17, 17, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389006805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389020702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 29, 26, 23, 24, 30, 25, 30, 30, 15, 15]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389034289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[2]R generation[0..12]: [8, 10, 16, 15, 13, 14, 17, 14, 17, 17, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389047666Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389061703Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.38907479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Generation requested: 18 >= found:18"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389145005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389170559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389184226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"":"downstairs","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389197383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389209421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"0","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389225847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"1","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389239474Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"client":"2","":"downstairs","session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389253071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389265379Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"85bc9fc2-3d7a-46c3-80df-f0aa7ab79576 is now active with session: 783cd92f-ad3b-4748-bb8b-dd511545a423","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389278206Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389290134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256,"session_id":"783cd92f-ad3b-4748-bb8b-dd511545a423"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:11.389302291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 32768 data: 3 4 4 3 3 2 2 2 02/30 Flush 03/30 Write block 535 len 32768 data: 2 2 5 5 5 5 5 5 04/30 Write block 409 len 16384 data: 2 3 3 3 05/30 Write block 574 len 4096 data: 5 06/30 Write block 556 len 24576 data: 2 2 2 2 2 2 07/30 Read block 232 len 28672 08/30 Read block 495 len 20480 09/30 Flush 10/30 Write block 27 len 40960 data: 4 4 3 3 2 2 2 3 3 5 11/30 Read block 368 len 36864 12/30 Write block 404 len 40960 data: 2 2 2 2 2 3 4 4 4 3 13/30 Write block 145 len 4096 data: 3 14/30 Write block 218 len 28672 data: 3 3 3 3 3 3 3 15/30 Write block 139 len 24576 data: 3 4 4 4 4 3 16/30 Read block 340 len 4096 17/30 Write block 496 len 8192 data: 3 2 18/30 Flush 19/30 Write block 184 len 40960 data: 3 3 3 4 4 3 3 3 4 4 20/30 Write block 108 len 16384 data: 2 2 2 2 21/30 Write block 444 len 12288 data: 3 4 4 22/30 Read block 509 len 40960 23/30 Write block 414 len 32768 data: 4 4 3 4 4 3 3 3 24/30 Write block 182 len 24576 data: 3 3 4 4 4 5 25/30 Write block 134 len 12288 data: 4 3 3 26/30 Flush 27/30 Write block 205 len 4096 data: 2 28/30 Read block 542 len 12288 29/30 Write block 531 len 20480 data: 4 3 3 2 3 30/30 Write block 466 len 28672 data: 3 3 3 2 3 3 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:11.407292774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1256} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 18 18 18 33 33 33 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 18 18 18 34 34 34 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 18 18 18 33 33 33 T T T 11 220-239 18 18 18 33 33 33 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 18 18 18 34 34 34 F F F 23 460-479 13 13 13 23 23 23 T T T 24 480-499 18 18 18 33 33 33 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 18 18 18 32 32 32 T T T 27 540-559 18 18 18 32 32 32 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 18, Max flush: 34 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 18 18 18 33 33 33 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 18 18 18 34 34 34 F F F 7 140-159 18 18 18 33 33 33 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 19 19 19 35 35 35 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 18 18 18 34 34 34 F F F 23 460-479 19 19 19 35 35 35 F F F 24 480-499 18 18 18 33 33 33 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 19 19 19 35 35 35 F F F 27 540-559 18 18 18 32 32 32 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 19, Max flush: 35 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.662253624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.705136315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.705770447Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.705814628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.705830694Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.705845591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} {"msg":"Crucible daeb2c3b-9492-4e14-927c-93a0cd00b64c has session id: 5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.706481393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Upstairs opts: Upstairs UUID: daeb2c3b-9492-4e14-927c-93a0cd00b64c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.706516845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Crucible stats registered with UUID: daeb2c3b-9492-4e14-927c-93a0cd00b64c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.706531832Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:15.706544739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.706940054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.706972547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} {"msg":"daeb2c3b-9492-4e14-927c-93a0cd00b64c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707066317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.70709398Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707127523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707156517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707185681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707211845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707296597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.70732841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707354074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707380809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34566)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707408332Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:56869)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.707482876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48471)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.7075147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.70802067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.708049993Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.708211378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.708676668Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.708704362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.708723658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709292364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709323098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[0]R flush_numbers[0..12]: [13, 33, 29, 26, 23, 34, 34, 33, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709346903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[0]R generation[0..12]: [8, 18, 16, 15, 13, 18, 18, 18, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709363769Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709377936Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[1]R flush_numbers[0..12]: [13, 33, 29, 26, 23, 34, 34, 33, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709391463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[1]R generation[0..12]: [8, 18, 16, 15, 13, 18, 18, 18, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.70940461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709418327Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[2]R flush_numbers[0..12]: [13, 33, 29, 26, 23, 34, 34, 33, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709431564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[2]R generation[0..12]: [8, 18, 16, 15, 13, 18, 18, 18, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709444471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709457748Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709479124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Generation requested: 20 >= found:20","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709508888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709533302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709546949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"":"downstairs","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709559746Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709571504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"0","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709584181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"1","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709596848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"client":"2","":"downstairs","session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709609356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709621153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"daeb2c3b-9492-4e14-927c-93a0cd00b64c is now active with session: 5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709633581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709644958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264,"session_id":"5b57fab4-6ade-47f4-b67c-6fe4ab44c0e5"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:15.709656805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 145 len 8192 data: 4 4 02/30 Write block 63 len 28672 data: 4 3 3 3 3 3 2 03/30 Write block 20 len 16384 data: 3 3 3 3 04/30 Read block 301 len 16384 05/30 Read block 492 len 24576 06/30 Read block 260 len 24576 07/30 Read block 294 len 28672 08/30 Read block 477 len 28672 09/30 Read block 21 len 12288 10/30 Read block 334 len 32768 11/30 Read block 272 len 20480 12/30 Flush 13/30 Write block 106 len 28672 data: 2 2 3 3 3 3 2 14/30 Write block 338 len 12288 data: 3 3 4 15/30 Read block 418 len 12288 16/30 Read block 394 len 16384 17/30 Read block 91 len 20480 18/30 Read block 450 len 20480 19/30 Read block 508 len 24576 20/30 Write block 579 len 32768 data: 3 2 2 2 2 2 2 2 21/30 Flush 22/30 Write block 540 len 4096 data: 6 23/30 Read block 417 len 8192 24/30 Read block 257 len 24576 25/30 Flush 26/30 Write block 61 len 12288 data: 5 5 5 27/30 Write block 528 len 28672 data: 4 5 4 5 4 4 3 28/30 Write block 37 len 40960 data: 5 5 5 5 5 3 2 2 3 4 29/30 Read block 91 len 12288 30/30 Read block 19 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:15.729149803Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1264} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 20 20 20 36 36 36 T T T 2 040-059 16 16 16 29 29 29 T T T 3 060-079 20 20 20 36 36 36 T T T 4 080-099 13 13 13 23 23 23 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 18 18 18 34 34 34 F F F 7 140-159 20 20 20 36 36 36 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 19 19 19 35 35 35 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 18 18 18 34 34 34 F F F 23 460-479 19 19 19 35 35 35 F F F 24 480-499 18 18 18 33 33 33 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 19 19 19 35 35 35 T T T 27 540-559 20 20 20 38 38 38 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 20 20 20 37 37 37 F F F Max gen: 20, Max flush: 38 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 21 21 21 39 39 39 F F F 2 040-059 21 21 21 39 39 39 F F F 3 060-079 21 21 21 39 39 39 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 18 18 18 34 34 34 F F F 7 140-159 20 20 20 36 36 36 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 19 19 19 35 35 35 F F F 11 220-239 18 18 18 33 33 33 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 18 18 18 34 34 34 F F F 23 460-479 19 19 19 35 35 35 F F F 24 480-499 18 18 18 33 33 33 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 21 21 21 39 39 39 F F F 27 540-559 20 20 20 38 38 38 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 20 20 20 37 37 37 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:19.961581892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.004833743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.005503718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.005546088Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.005561215Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.005577192Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} {"msg":"Crucible 7a56c948-3f4d-4498-954e-93dd47be9546 has session id: ae79fe78-46f0-4db8-95a8-66cf15502e1e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006106187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Upstairs opts: Upstairs UUID: 7a56c948-3f4d-4498-954e-93dd47be9546, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006163935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Crucible stats registered with UUID: 7a56c948-3f4d-4498-954e-93dd47be9546","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.00618535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:20.006199207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006633163Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006656098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} {"msg":"7a56c948-3f4d-4498-954e-93dd47be9546 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006726393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006745229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006768354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006782501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006797197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006820722Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006836599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006849936Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006901075Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.006934018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:20.007027438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59580)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.00706348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:20.007094693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54731)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.007121627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:20.007156699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.007182464Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.00766285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.007690654Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.008298462Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.008326636Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.008850293Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:20.008883676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.008004429Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51650)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.00809169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.009381731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010045947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[0]R flush_numbers[0..12]: [13, 39, 39, 39, 23, 37, 34, 36, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010641438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[0]R generation[0..12]: [8, 21, 21, 21, 13, 20, 18, 20, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010666352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010682129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[1]R flush_numbers[0..12]: [13, 39, 39, 39, 23, 37, 34, 36, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010700435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[1]R generation[0..12]: [8, 21, 21, 21, 13, 20, 18, 20, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010714642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010728759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[2]R flush_numbers[0..12]: [13, 39, 39, 39, 23, 37, 34, 36, 30, 34, 35, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010742196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[2]R generation[0..12]: [8, 21, 21, 21, 13, 20, 18, 20, 17, 18, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010755673Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.01076948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010782347Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010795435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010808362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010827977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"":"downstairs","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010841195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010852932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"0","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010866179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"1","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010879117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"client":"2","":"downstairs","session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010894063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010906251Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"7a56c948-3f4d-4498-954e-93dd47be9546 is now active with session: ae79fe78-46f0-4db8-95a8-66cf15502e1e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010918758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010930455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272,"session_id":"ae79fe78-46f0-4db8-95a8-66cf15502e1e"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:30.010942303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 488 len 20480 data: 2 2 2 2 2 02/30 Read block 530 len 28672 03/30 Read block 247 len 16384 04/30 Read block 57 len 16384 05/30 Read block 545 len 32768 06/30 Read block 426 len 40960 07/30 Read block 122 len 28672 08/30 Read block 432 len 8192 09/30 Write block 156 len 16384 data: 3 3 3 4 10/30 Read block 317 len 4096 11/30 Read block 235 len 4096 12/30 Read block 406 len 36864 13/30 Write block 368 len 36864 data: 2 2 2 2 2 3 3 4 4 14/30 Write block 202 len 40960 data: 2 2 2 3 3 3 3 3 3 3 15/30 Read block 421 len 20480 16/30 Write block 231 len 20480 data: 2 2 2 2 2 17/30 Read block 436 len 20480 18/30 Write block 519 len 16384 data: 2 3 4 4 19/30 Read block 427 len 20480 20/30 Read block 240 len 12288 21/30 Read block 38 len 4096 22/30 Write block 450 len 16384 data: 4 4 3 4 23/30 Write block 133 len 32768 data: 4 5 4 4 2 2 4 5 24/30 Flush 25/30 Write block 422 len 16384 data: 2 2 2 2 26/30 Read block 222 len 20480 27/30 Write block 502 len 36864 data: 4 4 3 3 3 3 3 3 3 28/30 Write block 38 len 4096 data: 6 29/30 Read block 234 len 40960 30/30 Write block 54 len 36864 data: 2 2 3 3 4 4 5 6 6 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:30.033288532Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1272} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 21 21 21 39 39 39 T T T 2 040-059 21 21 21 39 39 39 T T T 3 060-079 21 21 21 39 39 39 T T T 4 080-099 13 13 13 23 23 23 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 40 40 40 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 18 18 18 34 34 34 T T T 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 35 35 35 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 22 22 22 40 40 40 T T T 26 520-539 22 22 22 40 40 40 F F F 27 540-559 20 20 20 38 38 38 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 20 20 20 37 37 37 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 23 23 23 41 41 41 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 23 23 23 41 41 41 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 40 40 40 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 16 16 16 27 27 27 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 12 12 12 22 22 22 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 35 35 35 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 22 22 22 40 40 40 F F F 27 540-559 20 20 20 38 38 38 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 20 20 20 37 37 37 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.276854281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:34.318487002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.318917679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.349586165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35021208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.350246832Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.350261119Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.350276276Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} {"msg":"Crucible 452c81f4-ea36-4f8b-9729-4dfa35146d7a has session id: 3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.350912348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Upstairs opts: Upstairs UUID: 452c81f4-ea36-4f8b-9729-4dfa35146d7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35094992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Crucible stats registered with UUID: 452c81f4-ea36-4f8b-9729-4dfa35146d7a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.350966036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:34.350979353Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351421288Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35145717Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} {"msg":"452c81f4-ea36-4f8b-9729-4dfa35146d7a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351523496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.3515509Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351575544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351590241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351607357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351691519Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351721963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351749297Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35178197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351809894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:56444)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351833439Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37171)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351857853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46744)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.351931857Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.352529708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.352570849Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.352663329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.353151913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.353224077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.353290483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.353782776Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35390334Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[0]R flush_numbers[0..12]: [13, 41, 41, 41, 23, 37, 40, 40, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.353977454Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[0]R generation[0..12]: [8, 23, 23, 23, 13, 20, 22, 22, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35399696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354011517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[1]R flush_numbers[0..12]: [13, 41, 41, 41, 23, 37, 40, 40, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354024894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[1]R generation[0..12]: [8, 23, 23, 23, 13, 20, 22, 22, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354038131Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354051888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[2]R flush_numbers[0..12]: [13, 41, 41, 41, 23, 37, 40, 40, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354065525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[2]R generation[0..12]: [8, 23, 23, 23, 13, 20, 22, 22, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354078922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354095998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354117564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:38:34.354131951Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354153676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354166253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"":"downstairs","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.35417912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354190748Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"0","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354204115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"1","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354216802Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"client":"2","":"downstairs","session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.3542297Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354241797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"452c81f4-ea36-4f8b-9729-4dfa35146d7a is now active with session: 3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354254204Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354265921Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280,"session_id":"3e0d2c83-6d8c-4d5c-afed-3246ce87e0f1"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:34.354277689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 24576 data: 5 5 3 3 5 6 02/30 Flush 03/30 Write block 273 len 40960 data: 4 3 3 4 4 4 4 4 4 4 04/30 Write block 579 len 24576 data: 4 3 3 3 3 3 05/30 Read block 568 len 8192 06/30 Flush 07/30 Write block 340 len 20480 data: 5 2 2 3 3 08/30 Flush 09/30 Write block 401 len 24576 data: 4 3 3 3 3 3 10/30 Read block 310 len 20480 11/30 Read block 44 len 24576 12/30 Read block 242 len 16384 13/30 Flush 14/30 Write block 48 len 12288 data: 4 4 4 15/30 Write block 261 len 40960 data: 4 3 3 3 3 3 3 4 3 3 16/30 Read block 277 len 36864 17/30 Write block 33 len 40960 data: 3 4 4 6 6 7 6 6 6 4 18/30 Write block 55 len 32768 data: 3 4 4 5 5 6 7 7 19/30 Read block 171 len 8192 20/30 Read block 224 len 28672 21/30 Read block 332 len 12288 22/30 Write block 60 len 32768 data: 7 8 8 6 4 4 4 4 23/30 Write block 384 len 8192 data: 3 2 24/30 Write block 476 len 40960 data: 2 2 2 2 3 3 3 4 5 4 25/30 Read block 311 len 8192 26/30 Write block 112 len 20480 data: 3 2 3 4 4 27/30 Read block 549 len 36864 28/30 Read block 71 len 12288 29/30 Write block 440 len 24576 data: 3 3 4 3 4 5 30/30 Read block 396 len 24576 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:34.375097369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1280} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 23 23 23 41 41 41 T T T 2 040-059 23 23 23 41 41 41 T T T 3 060-079 23 23 23 41 41 41 T T T 4 080-099 13 13 13 23 23 23 F F F 5 100-119 20 20 20 37 37 37 T T T 6 120-139 24 24 24 42 42 42 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 24 24 24 43 43 43 T T T 14 280-299 24 24 24 43 43 43 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 12 12 12 22 22 22 T T T 20 400-419 24 24 24 45 45 45 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 22 22 22 40 40 40 T T T 23 460-479 19 19 19 35 35 35 T T T 24 480-499 22 22 22 40 40 40 T T T 25 500-519 23 23 23 41 41 41 F F F 26 520-539 22 22 22 40 40 40 F F F 27 540-559 20 20 20 38 38 38 F F F 28 560-579 24 24 24 43 43 43 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 24, Max flush: 45 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 25 25 25 46 46 46 F F F 2 040-059 25 25 25 46 46 46 F F F 3 060-079 25 25 25 46 46 46 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 25 25 25 46 46 46 F F F 6 120-139 24 24 24 42 42 42 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 12 12 12 19 19 19 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 25 25 25 46 46 46 F F F 20 400-419 24 24 24 45 45 45 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 25 25 25 46 46 46 F F F 24 480-499 25 25 25 46 46 46 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 22 22 22 40 40 40 F F F 27 540-559 20 20 20 38 38 38 F F F 28 560-579 24 24 24 43 43 43 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 25, Max flush: 46 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.664023931Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.708404557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.709335856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.709380216Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.709395233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.70940963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} {"msg":"Crucible 6435a1c9-f223-47d2-8756-2a1d92bf13b8 has session id: 0a58a1b4-a35f-4325-bc3e-791a482009b7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.709992713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Upstairs opts: Upstairs UUID: 6435a1c9-f223-47d2-8756-2a1d92bf13b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710034484Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Crucible stats registered with UUID: 6435a1c9-f223-47d2-8756-2a1d92bf13b8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.71005555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:38.710069337Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710511341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.7105627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} {"msg":"6435a1c9-f223-47d2-8756-2a1d92bf13b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710590514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710604031Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710626286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710640513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.71065591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710679235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710705239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710727174Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710752519Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.710778083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36561)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.71097723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63736)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.711018821Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:38.711059132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:38.711095335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:38.711128357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.711240193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.71176251Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.711792344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.712365559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.712494451Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.712953152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:38.713057949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.712936339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59148)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.713125539Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.714036521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.71455188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[0]R flush_numbers[0..12]: [13, 46, 46, 46, 23, 46, 42, 42, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715131554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[0]R generation[0..12]: [8, 25, 25, 25, 13, 25, 24, 24, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.7151515Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715167956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[1]R flush_numbers[0..12]: [13, 46, 46, 46, 23, 46, 42, 42, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715181503Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[1]R generation[0..12]: [8, 25, 25, 25, 13, 25, 24, 24, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.71519486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715208767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[2]R flush_numbers[0..12]: [13, 46, 46, 46, 23, 46, 42, 42, 30, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715221864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[2]R generation[0..12]: [8, 25, 25, 25, 13, 25, 24, 24, 17, 18, 22, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715234922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715248269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715261016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715274243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.71528694Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715299298Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"":"downstairs","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715312175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715327901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"0","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715341509Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"1","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:38:48.715354146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"client":"2","":"downstairs","session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715373392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715385169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"6435a1c9-f223-47d2-8756-2a1d92bf13b8 is now active with session: 0a58a1b4-a35f-4325-bc3e-791a482009b7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715397736Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715409284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288,"session_id":"0a58a1b4-a35f-4325-bc3e-791a482009b7"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:48.715420961Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 32768 data: 5 5 4 3 2 2 2 2 02/30 Read block 458 len 8192 03/30 Write block 545 len 16384 data: 4 3 3 4 04/30 Read block 223 len 24576 05/30 Write block 539 len 24576 data: 6 7 6 6 6 6 06/30 Read block 298 len 4096 07/30 Read block 436 len 20480 08/30 Write block 559 len 12288 data: 3 3 3 09/30 Read block 302 len 32768 10/30 Write block 562 len 4096 data: 2 11/30 Read block 356 len 24576 12/30 Read block 32 len 12288 13/30 Write block 556 len 20480 data: 3 3 3 4 4 14/30 Write block 226 len 36864 data: 2 2 2 2 2 3 3 3 3 15/30 Write block 240 len 20480 data: 3 3 3 2 2 16/30 Read block 114 len 16384 17/30 Write block 395 len 32768 data: 3 2 2 3 3 3 5 4 18/30 Write block 472 len 4096 data: 4 19/30 Flush 20/30 Write block 119 len 28672 data: 4 4 4 3 2 2 2 21/30 Write block 494 len 28672 data: 3 3 4 3 3 2 4 22/30 Read block 518 len 20480 23/30 Read block 117 len 12288 24/30 Write block 317 len 32768 data: 2 2 2 3 3 2 2 2 25/30 Read block 378 len 16384 26/30 Read block 93 len 40960 27/30 Write block 250 len 8192 data: 2 2 28/30 Read block 128 len 16384 29/30 Write block 8 len 28672 data: 2 3 3 3 3 4 4 30/30 Read block 186 len 40960 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 9 11 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:48.736878563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1288} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 T T T 1 020-039 25 25 25 46 46 46 F F F 2 040-059 25 25 25 46 46 46 F F F 3 060-079 26 26 26 47 47 47 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 25 25 25 46 46 46 T T T 6 120-139 24 24 24 42 42 42 T T T 7 140-159 24 24 24 42 42 42 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 26 26 26 47 47 47 F F F 12 240-259 26 26 26 47 47 47 T T T 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 12 12 12 19 19 19 T T T 16 320-339 20 20 20 37 37 37 T T T 17 340-359 24 24 24 44 44 44 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 26 26 26 47 47 47 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 26 26 26 47 47 47 F F F 24 480-499 25 25 25 46 46 46 T T T 25 500-519 23 23 23 41 41 41 T T T 26 520-539 26 26 26 47 47 47 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 26, Max flush: 47 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 48 48 48 F F F 1 020-039 25 25 25 46 46 46 F F F 2 040-059 25 25 25 46 46 46 F F F 3 060-079 26 26 26 47 47 47 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 27 27 27 48 48 48 F F F 6 120-139 27 27 27 48 48 48 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 26 26 26 47 47 47 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 25 25 25 46 46 46 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 27 27 27 48 48 48 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 24 24 24 44 44 44 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 26 26 26 47 47 47 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 26 26 26 47 47 47 F F F 24 480-499 27 27 27 48 48 48 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 26 26 26 47 47 47 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 27, Max flush: 48 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:52.996475629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.039775819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.040550962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.04060537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.040625115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.040644471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} {"msg":"Crucible 3317cdbc-8305-4bcc-ae89-274592aa1232 has session id: 1d924a1e-87db-412c-b01d-629f91f504be","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041207149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Upstairs opts: Upstairs UUID: 3317cdbc-8305-4bcc-ae89-274592aa1232, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041247641Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Crucible stats registered with UUID: 3317cdbc-8305-4bcc-ae89-274592aa1232","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041265397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:53.041279614Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041700653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041736555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} {"msg":"3317cdbc-8305-4bcc-ae89-274592aa1232 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041799771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041829085Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041853089Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041868226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041882873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041905848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041932852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041957937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.041984301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042037929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"ds_connection connected from Ok(127.0.0.1:56503)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042114833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"ds_connection connected from Ok(127.0.0.1:41872)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042152375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"ds_connection connected from Ok(127.0.0.1:60141)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042182338Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042765772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.042930536Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.043367372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.043549832Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.043589284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.04397551Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044201861Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044272965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[0]R flush_numbers[0..12]: [48, 46, 46, 47, 23, 48, 48, 42, 30, 34, 40, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044770148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[0]R generation[0..12]: [27, 25, 25, 26, 13, 27, 27, 24, 17, 18, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044798011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044813378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[1]R flush_numbers[0..12]: [48, 46, 46, 47, 23, 48, 48, 42, 30, 34, 40, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044827465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[1]R generation[0..12]: [27, 25, 25, 26, 13, 27, 27, 24, 17, 18, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044840972Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044854449Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[2]R flush_numbers[0..12]: [48, 46, 46, 47, 23, 48, 48, 42, 30, 34, 40, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044867676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[2]R generation[0..12]: [27, 25, 25, 26, 13, 27, 27, 24, 17, 18, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044881094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044894761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044911697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044925294Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.04494398Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044956777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"":"downstairs","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044969894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044981872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"0","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.044995179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"1","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045008396Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"client":"2","":"downstairs","session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045021383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045033591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"3317cdbc-8305-4bcc-ae89-274592aa1232 is now active with session: 1d924a1e-87db-412c-b01d-629f91f504be","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045046158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045057725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296,"session_id":"1d924a1e-87db-412c-b01d-629f91f504be"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:53.045069493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 28672 data: 3 2 3 4 5 5 6 02/30 Read block 457 len 12288 03/30 Read block 478 len 12288 04/30 Write block 147 len 32768 data: 4 3 3 3 3 3 3 2 05/30 Write block 80 len 28672 data: 2 2 3 3 4 4 3 06/30 Read block 246 len 20480 07/30 Read block 375 len 36864 08/30 Read block 209 len 8192 09/30 Flush 10/30 Write block 307 len 16384 data: 2 3 3 4 11/30 Flush 12/30 Write block 238 len 24576 data: 3 3 4 4 4 3 13/30 Read block 369 len 8192 14/30 Read block 97 len 4096 15/30 Read block 568 len 20480 16/30 Read block 159 len 40960 17/30 Write block 271 len 20480 data: 4 4 5 4 4 18/30 Read block 271 len 36864 19/30 Read block 19 len 16384 20/30 Read block 101 len 12288 21/30 Write block 235 len 12288 data: 3 2 2 22/30 Read block 39 len 32768 23/30 Write block 481 len 4096 data: 4 24/30 Read block 483 len 28672 25/30 Read block 87 len 16384 26/30 Read block 247 len 24576 27/30 Write block 341 len 12288 data: 3 3 4 28/30 Write block 82 len 24576 data: 4 4 5 5 4 3 29/30 Read block 423 len 32768 30/30 Write block 149 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 18 18 16 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:53.064821354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1296} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 48 48 48 F F F 1 020-039 25 25 25 46 46 46 F F F 2 040-059 28 28 28 49 49 49 F F F 3 060-079 26 26 26 47 47 47 F F F 4 080-099 28 28 28 49 49 49 T T T 5 100-119 27 27 27 48 48 48 F F F 6 120-139 27 27 27 48 48 48 F F F 7 140-159 28 28 28 49 49 49 T T T 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 26 26 26 47 47 47 T T T 12 240-259 27 27 27 48 48 48 T T T 13 260-279 25 25 25 46 46 46 T T T 14 280-299 24 24 24 43 43 43 F F F 15 300-319 28 28 28 50 50 50 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 24 24 24 44 44 44 T T T 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 26 26 26 47 47 47 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 26 26 26 47 47 47 F F F 24 480-499 27 27 27 48 48 48 T T T 25 500-519 27 27 27 48 48 48 F F F 26 520-539 26 26 26 47 47 47 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 28, Max flush: 50 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 48 48 48 F F F 1 020-039 25 25 25 46 46 46 F F F 2 040-059 28 28 28 49 49 49 F F F 3 060-079 26 26 26 47 47 47 F F F 4 080-099 29 29 29 51 51 51 F F F 5 100-119 27 27 27 48 48 48 F F F 6 120-139 27 27 27 48 48 48 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 18 18 18 34 34 34 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 28 28 28 50 50 50 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 29 29 29 51 51 51 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 26 26 26 47 47 47 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 26 26 26 47 47 47 F F F 24 480-499 29 29 29 51 51 51 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 26 26 26 47 47 47 F F F 27 540-559 26 26 26 47 47 47 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 29, Max flush: 51 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.314852373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.357914165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.358569823Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.358613843Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.35862925Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.358645166Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} {"msg":"Crucible 73ffbbed-cbec-4f4f-b97b-8a80d07629fc has session id: abd7ed02-ec72-4ded-8558-d1b93f765ae2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359287167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Upstairs opts: Upstairs UUID: 73ffbbed-cbec-4f4f-b97b-8a80d07629fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359328118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Crucible stats registered with UUID: 73ffbbed-cbec-4f4f-b97b-8a80d07629fc","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359344575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:57.359358312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359798147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359879059Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} {"msg":"73ffbbed-cbec-4f4f-b97b-8a80d07629fc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359910212Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359925179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359945375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359960312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.359977738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360002092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360027127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360049322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360076396Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.36010761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40578)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360134554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34731)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360177914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51472)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360373172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360738483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.360767067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.361388282Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.361416166Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.362046809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.362081472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.362422288Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.362903474Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[0]R flush_numbers[0..12]: [48, 46, 49, 47, 51, 48, 48, 51, 30, 34, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363373362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[0]R generation[0..12]: [27, 25, 28, 26, 29, 27, 27, 29, 17, 18, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363397397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363412324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[1]R flush_numbers[0..12]: [48, 46, 49, 47, 51, 48, 48, 51, 30, 34, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.36342953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[1]R generation[0..12]: [27, 25, 28, 26, 29, 27, 27, 29, 17, 18, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363443357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363457234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[2]R flush_numbers[0..12]: [48, 46, 49, 47, 51, 48, 48, 51, 30, 34, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363470781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[2]R generation[0..12]: [27, 25, 28, 26, 29, 27, 27, 29, 17, 18, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363484378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:38:57.363498425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363514162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363527299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363539936Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363552423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"":"downstairs","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363565141Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363576858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"0","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363589685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"1","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363602483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"client":"2","":"downstairs","session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363617129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363629387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"73ffbbed-cbec-4f4f-b97b-8a80d07629fc is now active with session: abd7ed02-ec72-4ded-8558-d1b93f765ae2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363642114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363653681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304,"session_id":"abd7ed02-ec72-4ded-8558-d1b93f765ae2"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:38:57.363665429Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 45 len 32768 data: 4 5 4 5 5 5 3 4 02/30 Write block 25 len 16384 data: 3 3 5 5 03/30 Write block 22 len 28672 data: 4 4 3 4 4 6 6 04/30 Write block 407 len 12288 data: 3 3 4 05/30 Read block 253 len 20480 06/30 Read block 201 len 36864 07/30 Write block 591 len 32768 data: 2 3 3 2 2 2 2 2 08/30 Read block 314 len 24576 09/30 Write block 340 len 28672 data: 6 4 4 5 4 4 4 10/30 Read block 574 len 28672 11/30 Write block 467 len 20480 data: 4 4 3 4 4 12/30 Write block 530 len 24576 data: 5 6 5 5 4 4 13/30 Write block 537 len 40960 data: 6 6 7 8 7 7 7 7 5 4 14/30 Read block 226 len 24576 15/30 Write block 539 len 12288 data: 8 9 8 16/30 Write block 104 len 12288 data: 4 3 3 17/30 Write block 294 len 12288 data: 3 3 3 18/30 Write block 328 len 12288 data: 5 4 4 19/30 Read block 16 len 4096 20/30 Read block 1 len 16384 21/30 Read block 589 len 8192 22/30 Read block 14 len 24576 23/30 Read block 511 len 32768 24/30 Write block 190 len 4096 data: 4 25/30 Write block 529 len 24576 data: 6 6 7 6 6 5 26/30 Read block 468 len 40960 27/30 Write block 423 len 36864 data: 3 3 3 2 2 2 2 2 2 28/30 Write block 178 len 28672 data: 5 3 4 3 4 4 5 29/30 Write block 342 len 4096 data: 5 30/30 Write block 57 len 16384 data: 6 7 6 8 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:38:57.381505554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1304} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 48 48 48 F F F 1 020-039 25 25 25 46 46 46 T T T 2 040-059 28 28 28 49 49 49 T T T 3 060-079 26 26 26 47 47 47 T T T 4 080-099 29 29 29 51 51 51 F F F 5 100-119 27 27 27 48 48 48 T T T 6 120-139 27 27 27 48 48 48 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 17 17 17 30 30 30 T T T 9 180-199 18 18 18 34 34 34 T T T 10 200-219 22 22 22 40 40 40 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 24 24 24 43 43 43 T T T 15 300-319 28 28 28 50 50 50 F F F 16 320-339 27 27 27 48 48 48 T T T 17 340-359 29 29 29 51 51 51 T T T 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 26 26 26 47 47 47 T T T 21 420-439 23 23 23 41 41 41 T T T 22 440-459 25 25 25 46 46 46 F F F 23 460-479 26 26 26 47 47 47 T T T 24 480-499 29 29 29 51 51 51 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 26 26 26 47 47 47 T T T 27 540-559 26 26 26 47 47 47 T T T 28 560-579 26 26 26 47 47 47 F F F 29 580-599 24 24 24 43 43 43 T T T Max gen: 29, Max flush: 51 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 48 48 48 F F F 1 020-039 30 30 30 52 52 52 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 30 30 30 52 52 52 F F F 4 080-099 29 29 29 51 51 51 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 27 27 27 48 48 48 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 30 30 30 52 52 52 F F F 15 300-319 28 28 28 50 50 50 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 29 29 29 51 51 51 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 30, Max flush: 52 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.675300653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.7184751Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.71916934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719210731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719226487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719240874Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} {"msg":"Crucible 3a154a86-916f-476f-a009-f087d1bfbdb5 has session id: c5a021d4-46ef-49e5-87b1-8c82960ea694","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719761961Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Upstairs opts: Upstairs UUID: 3a154a86-916f-476f-a009-f087d1bfbdb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719807051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Crucible stats registered with UUID: 3a154a86-916f-476f-a009-f087d1bfbdb5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.719825358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:01.719837935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.72027796Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720301954Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} {"msg":"3a154a86-916f-476f-a009-f087d1bfbdb5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720373279Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720392525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.7204139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720445093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720472407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720498562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720526166Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.72055144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720579044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720603539Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"ds_connection connected from Ok(127.0.0.1:36011)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720672704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"ds_connection connected from Ok(127.0.0.1:43946)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720712526Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"ds_connection connected from Ok(127.0.0.1:53784)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.720744328Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.721390648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.721418222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.722083158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.722113322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.722627201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.722733278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.722793415Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723235919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[0]R flush_numbers[0..12]: [48, 52, 52, 52, 51, 52, 48, 51, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723718344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[0]R generation[0..12]: [27, 30, 30, 30, 29, 30, 27, 29, 30, 30, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.72373687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723751187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[1]R flush_numbers[0..12]: [48, 52, 52, 52, 51, 52, 48, 51, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723764734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[1]R generation[0..12]: [27, 30, 30, 30, 29, 30, 27, 29, 30, 30, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723777891Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723791248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[2]R flush_numbers[0..12]: [48, 52, 52, 52, 51, 52, 48, 51, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723804276Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[2]R generation[0..12]: [27, 30, 30, 30, 29, 30, 27, 29, 30, 30, 22, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723817273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723834469Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Max found gen is 31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723847336Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Generation requested: 32 >= found:31","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723860284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723878689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723891367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"":"downstairs","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723904064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723915881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"0","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723928909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"1","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723941686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"client":"2","":"downstairs","session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723954203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.72396982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"3a154a86-916f-476f-a009-f087d1bfbdb5 is now active with session: c5a021d4-46ef-49e5-87b1-8c82960ea694","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723982467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.723994064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312,"session_id":"c5a021d4-46ef-49e5-87b1-8c82960ea694"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:01.724006022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 8192 data: 3 3 02/30 Write block 4 len 20480 data: 2 2 2 2 3 03/30 Read block 422 len 12288 04/30 Read block 342 len 36864 05/30 Read block 583 len 16384 06/30 Write block 300 len 28672 data: 2 2 2 3 3 3 3 07/30 Read block 365 len 32768 08/30 Write block 50 len 20480 data: 6 4 5 3 4 09/30 Flush 10/30 Write block 523 len 16384 data: 4 4 5 4 11/30 Read block 151 len 8192 12/30 Write block 480 len 20480 data: 4 5 4 5 6 13/30 Read block 125 len 24576 14/30 Write block 313 len 28672 data: 4 3 3 3 3 3 3 15/30 Write block 203 len 24576 data: 3 3 4 4 4 4 16/30 Write block 285 len 28672 data: 4 3 3 3 3 2 2 17/30 Read block 441 len 4096 18/30 Read block 48 len 16384 19/30 Read block 441 len 36864 20/30 Flush 21/30 Write block 291 len 32768 data: 3 2 2 4 4 4 3 2 22/30 Flush 23/30 Write block 256 len 20480 data: 4 4 4 4 3 24/30 Read block 313 len 8192 25/30 Write block 72 len 16384 data: 3 3 2 2 26/30 Read block 540 len 32768 27/30 Write block 199 len 16384 data: 2 2 2 3 28/30 Read block 201 len 32768 29/30 Read block 371 len 24576 30/30 Write block 120 len 12288 data: 5 5 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:01.743819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1312} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 53 53 53 F F F 1 020-039 30 30 30 52 52 52 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 30 30 30 52 52 52 T T T 4 080-099 29 29 29 51 51 51 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 27 27 27 48 48 48 T T T 7 140-159 29 29 29 51 51 51 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 30 30 30 52 52 52 T T T 10 200-219 32 32 32 54 54 54 T T T 11 220-239 29 29 29 51 51 51 F F F 12 240-259 29 29 29 51 51 51 T T T 13 260-279 29 29 29 51 51 51 T T T 14 280-299 32 32 32 55 55 55 F F F 15 300-319 32 32 32 54 54 54 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 32 32 32 53 53 53 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 32, Max flush: 55 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 53 53 53 F F F 1 020-039 30 30 30 52 52 52 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 33 33 33 56 56 56 F F F 4 080-099 29 29 29 51 51 51 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 33 33 33 56 56 56 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 33 33 33 56 56 56 F F F 10 200-219 33 33 33 56 56 56 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 33 33 33 56 56 56 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 32 32 32 54 54 54 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 32 32 32 53 53 53 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 33, Max flush: 56 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:05.986398933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:06.030596549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.031334469Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.063394354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064230323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064262826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064276393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.06429055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} {"msg":"Crucible da43d13f-78ab-4e87-a252-eb3a4f2fd2df has session id: 74b43571-365b-432a-9320-4a2202009f8c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064849839Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Upstairs opts: Upstairs UUID: da43d13f-78ab-4e87-a252-eb3a4f2fd2df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064895979Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Crucible stats registered with UUID: da43d13f-78ab-4e87-a252-eb3a4f2fd2df","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.064918824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:06.064940459Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065347921Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065375025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} {"msg":"da43d13f-78ab-4e87-a252-eb3a4f2fd2df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065434642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065452428Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065476963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065501538Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065527332Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065549557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065573922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065594348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065618162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56499)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065648316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.06567625Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62083)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065703464Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34015)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.065842034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.066245087Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.066374259Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.066699338Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.066731971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.066880889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067266825Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067436719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067462833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[0]R flush_numbers[0..12]: [53, 52, 53, 56, 51, 52, 56, 51, 52, 56, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067824635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[0]R generation[0..12]: [32, 30, 32, 33, 29, 30, 33, 29, 30, 33, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067849029Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067867905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[1]R flush_numbers[0..12]: [53, 52, 53, 56, 51, 52, 56, 51, 52, 56, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067882102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[1]R generation[0..12]: [32, 30, 32, 33, 29, 30, 33, 29, 30, 33, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067895779Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067910306Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[2]R flush_numbers[0..12]: [53, 52, 53, 56, 51, 52, 56, 51, 52, 56, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067924033Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[2]R generation[0..12]: [32, 30, 32, 33, 29, 30, 33, 29, 30, 33, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.06793735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067951227Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:39:06.067963934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.067979891Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068009344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068022512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"":"downstairs","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068035369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068047236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"0","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068060293Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"1","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068073201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"client":"2","":"downstairs","session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068088567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068100965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"da43d13f-78ab-4e87-a252-eb3a4f2fd2df is now active with session: 74b43571-365b-432a-9320-4a2202009f8c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068113502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068125179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320,"session_id":"74b43571-365b-432a-9320-4a2202009f8c"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:06.068136977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 186 len 28672 data: 5 6 5 4 5 4 5 02/30 Read block 186 len 16384 03/30 Read block 268 len 32768 04/30 Read block 137 len 32768 05/30 Read block 549 len 40960 06/30 Flush 07/30 Write block 391 len 4096 data: 3 08/30 Read block 592 len 28672 09/30 Write block 75 len 36864 data: 3 2 2 2 2 3 3 5 5 10/30 Write block 537 len 24576 data: 7 7 9 10 9 8 11/30 Read block 13 len 36864 12/30 Write block 309 len 32768 data: 4 5 4 4 5 4 4 4 13/30 Write block 131 len 40960 data: 3 4 5 6 6 6 4 4 6 7 14/30 Read block 412 len 28672 15/30 Read block 32 len 4096 16/30 Read block 549 len 12288 17/30 Write block 242 len 8192 data: 5 4 18/30 Write block 541 len 40960 data: 10 9 8 8 6 5 4 5 4 4 19/30 Flush 20/30 Write block 408 len 28672 data: 4 5 5 5 5 4 5 21/30 Read block 463 len 24576 22/30 Read block 116 len 36864 23/30 Write block 34 len 4096 data: 5 24/30 Read block 384 len 40960 25/30 Read block 4 len 40960 26/30 Write block 5 len 24576 data: 3 3 3 4 4 4 27/30 Read block 216 len 16384 28/30 Read block 93 len 24576 29/30 Read block 222 len 20480 30/30 Read block 447 len 32768 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 10 10 11 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:06.090407292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1320} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 53 53 53 T T T 1 020-039 30 30 30 52 52 52 T T T 2 040-059 32 32 32 53 53 53 F F F 3 060-079 34 34 34 58 58 58 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 33 33 33 56 56 56 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 30 30 30 52 52 52 T T T 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 34, Max flush: 58 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 59 59 59 F F F 1 020-039 35 35 35 59 59 59 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 34 34 34 58 58 58 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 33 33 33 56 56 56 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 27 27 27 48 48 48 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 35, Max flush: 59 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.354477371Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:10.396910829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.397432225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.428540847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429205133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429238926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429253662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429268159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} {"msg":"Crucible 78bcb465-3124-4891-be8f-6ab2f34bf967 has session id: 3ac320de-4207-4dc9-a564-9537c0c7372b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429880416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Upstairs opts: Upstairs UUID: 78bcb465-3124-4891-be8f-6ab2f34bf967, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429920918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Crucible stats registered with UUID: 78bcb465-3124-4891-be8f-6ab2f34bf967","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.429946862Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:10.429970597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430335628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430368001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} {"msg":"78bcb465-3124-4891-be8f-6ab2f34bf967 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430421899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430449733Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430479207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430495543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.43051002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430524057Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430537924Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.43055759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430575136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430589423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49432)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430623315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42082)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.43065035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57175)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.430674495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.431457455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.431486519Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.431586667Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432103705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432135618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432151315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432649247Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432754564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[0]R flush_numbers[0..12]: [59, 59, 53, 58, 58, 52, 58, 58, 52, 57, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432836706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[0]R generation[0..12]: [35, 35, 32, 34, 34, 30, 34, 34, 30, 34, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432855652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432870179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[1]R flush_numbers[0..12]: [59, 59, 53, 58, 58, 52, 58, 58, 52, 57, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432883686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[1]R generation[0..12]: [35, 35, 32, 34, 34, 30, 34, 34, 30, 34, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432897003Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.43291101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[2]R flush_numbers[0..12]: [59, 59, 53, 58, 58, 52, 58, 58, 52, 57, 56, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432924737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[2]R generation[0..12]: [35, 35, 32, 34, 34, 30, 34, 34, 30, 34, 33, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432938344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432951851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432964578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.432984214Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433022676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433038392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"":"downstairs","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433051409Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433063177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"0","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433076144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"1","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433088601Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"client":"2","":"downstairs","session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433101149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433112986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"78bcb465-3124-4891-be8f-6ab2f34bf967 is now active with session: 3ac320de-4207-4dc9-a564-9537c0c7372b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433125573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433137121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328,"session_id":"3ac320de-4207-4dc9-a564-9537c0c7372b"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:10.433148808Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 20480 data: 3 4 4 4 5 02/30 Read block 409 len 8192 03/30 Read block 374 len 20480 04/30 Write block 183 len 40960 data: 5 6 5 6 7 6 5 6 5 6 05/30 Read block 230 len 28672 06/30 Read block 559 len 24576 07/30 Write block 32 len 32768 data: 3 4 6 5 7 7 8 7 08/30 Read block 425 len 24576 09/30 Read block 189 len 36864 10/30 Flush 11/30 Write block 66 len 36864 data: 6 6 5 4 3 3 4 4 3 12/30 Write block 61 len 8192 data: 9 9 13/30 Read block 274 len 28672 14/30 Read block 468 len 28672 15/30 Write block 207 len 32768 data: 5 5 4 4 4 3 3 3 16/30 Write block 342 len 28672 data: 6 6 5 5 5 3 3 17/30 Read block 388 len 40960 18/30 Read block 315 len 40960 19/30 Write block 514 len 16384 data: 3 3 3 4 20/30 Read block 304 len 4096 21/30 Read block 47 len 32768 22/30 Read block 246 len 16384 23/30 Read block 31 len 4096 24/30 Read block 137 len 12288 25/30 Write block 207 len 8192 data: 6 6 26/30 Flush 27/30 Write block 342 len 36864 data: 7 7 6 6 6 4 4 4 4 28/30 Read block 477 len 24576 29/30 Write block 329 len 24576 data: 5 5 5 5 3 3 30/30 Write block 12 len 32768 data: 4 5 5 4 4 4 3 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:10.454522588Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1328} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 60 60 60 T T T 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 36 36 36 60 60 60 F F F 10 200-219 36 36 36 61 61 61 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 30 30 30 52 52 52 T T T 17 340-359 36 36 36 61 61 61 T T T 18 360-379 32 32 32 53 53 53 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 36, Max flush: 61 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 36 36 36 60 60 60 F F F 10 200-219 36 36 36 61 61 61 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 37 37 37 62 62 62 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 30 30 30 52 52 52 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.714549271Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:14.756050171Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.756452504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.787022921Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.787604305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.787646286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.787663872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.787682118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} {"msg":"Crucible 3edd1ea8-c011-4070-b3e9-47c4ac354166 has session id: d714fa8d-4ed5-47e9-9582-59e666d64782","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788258483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Upstairs opts: Upstairs UUID: 3edd1ea8-c011-4070-b3e9-47c4ac354166, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788304513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Crucible stats registered with UUID: 3edd1ea8-c011-4070-b3e9-47c4ac354166","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788332247Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:14.788354253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788799456Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.7888249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} {"msg":"3edd1ea8-c011-4070-b3e9-47c4ac354166 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.78887292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788890046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788913571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788927898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788942305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788955522Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788970209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.788989165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.78901187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789060129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"ds_connection connected from Ok(127.0.0.1:46898)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789085864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"ds_connection connected from Ok(127.0.0.1:48819)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789111838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"ds_connection connected from Ok(127.0.0.1:49882)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789144321Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789873723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789905606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.789941878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.790524772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.790562034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.790624461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791167903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791193687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[0]R flush_numbers[0..12]: [62, 60, 53, 61, 58, 52, 58, 58, 52, 60, 61, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791222161Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[0]R generation[0..12]: [37, 36, 32, 36, 34, 30, 34, 34, 30, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791237168Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791251245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[1]R flush_numbers[0..12]: [62, 60, 53, 61, 58, 52, 58, 58, 52, 60, 61, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791264682Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[1]R generation[0..12]: [37, 36, 32, 36, 34, 30, 34, 34, 30, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791277939Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791294206Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[2]R flush_numbers[0..12]: [62, 60, 53, 61, 58, 52, 58, 58, 52, 60, 61, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791307563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[2]R generation[0..12]: [37, 36, 32, 36, 34, 30, 34, 34, 30, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.79132108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791334957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791347724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791360521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791372879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791385466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791405752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791417339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"0","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791430566Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"1","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791443403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"client":"2","":"downstairs","session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.79145649Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791468798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"3edd1ea8-c011-4070-b3e9-47c4ac354166 is now active with session: d714fa8d-4ed5-47e9-9582-59e666d64782","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791481125Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791494562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336,"session_id":"d714fa8d-4ed5-47e9-9582-59e666d64782"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:14.791506659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 199 len 28672 data: 3 3 3 4 4 4 5 02/30 Read block 157 len 32768 03/30 Read block 523 len 36864 04/30 Read block 26 len 8192 05/30 Write block 8 len 24576 data: 6 5 5 4 5 6 06/30 Read block 28 len 24576 07/30 Read block 63 len 16384 08/30 Read block 372 len 4096 09/30 Write block 323 len 20480 data: 3 3 2 2 3 10/30 Read block 282 len 32768 11/30 Write block 436 len 16384 data: 3 3 4 4 12/30 Write block 392 len 20480 data: 3 3 3 4 3 13/30 Write block 146 len 36864 data: 5 5 4 5 5 4 4 4 3 14/30 Read block 201 len 8192 15/30 Read block 64 len 40960 16/30 Read block 233 len 8192 17/30 Write block 215 len 12288 data: 3 3 3 18/30 Read block 558 len 32768 19/30 Write block 565 len 32768 data: 3 3 4 4 4 3 3 4 20/30 Read block 217 len 24576 21/30 Read block 426 len 36864 22/30 Read block 188 len 32768 23/30 Write block 264 len 24576 data: 4 4 4 4 5 4 24/30 Read block 405 len 28672 25/30 Read block 405 len 24576 26/30 Read block 294 len 24576 27/30 Write block 575 len 32768 data: 4 4 3 3 5 4 4 4 28/30 Read block 281 len 12288 29/30 Read block 365 len 32768 30/30 Write block 225 len 8192 data: 2 3 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:14.81301655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1336} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 T T T 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 T T T 8 160-179 30 30 30 52 52 52 F F F 9 180-199 36 36 36 60 60 60 T T T 10 200-219 36 36 36 61 61 61 T T T 11 220-239 29 29 29 51 51 51 T T T 12 240-259 34 34 34 58 58 58 F F F 13 260-279 33 33 33 56 56 56 T T T 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 37 37 37 62 62 62 T T T 17 340-359 37 37 37 62 62 62 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 34 34 34 58 58 58 T T T 20 400-419 35 35 35 59 59 59 F F F 21 420-439 30 30 30 52 52 52 T T T 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 26 26 26 47 47 47 T T T 29 580-599 30 30 30 52 52 52 T T T Max gen: 37, Max flush: 62 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 38 38 38 63 63 63 F F F 10 200-219 38 38 38 63 63 63 F F F 11 220-239 38 38 38 63 63 63 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 32 32 32 55 55 55 F F F 15 300-319 34 34 34 58 58 58 F F F 16 320-339 38 38 38 63 63 63 F F F 17 340-359 37 37 37 62 62 62 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 38 38 38 63 63 63 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 25 25 25 46 46 46 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 38 38 38 63 63 63 F F F 29 580-599 38 38 38 63 63 63 F F F Max gen: 38, Max flush: 63 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.116643399Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:19.159522341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.160088688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.191019948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.191775714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.191810976Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.191826293Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19184051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} {"msg":"Crucible 21ddce00-1970-4307-bbab-55cc026cf6b3 has session id: a169a835-1710-4949-a0d5-3cef251ad182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.192448878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Upstairs opts: Upstairs UUID: 21ddce00-1970-4307-bbab-55cc026cf6b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.192494468Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Crucible stats registered with UUID: 21ddce00-1970-4307-bbab-55cc026cf6b3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.192521492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:19.192542828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.192914767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19294783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} {"msg":"21ddce00-1970-4307-bbab-55cc026cf6b3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193034191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193067344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193094498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193109405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193124862Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193138219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193152466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193165483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19317907Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193192687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"ds_connection connected from Ok(127.0.0.1:45029)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193214322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"ds_connection connected from Ok(127.0.0.1:53806)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.193284187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"ds_connection connected from Ok(127.0.0.1:44297)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19331816Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194123156Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194156058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194173354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194764557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194865395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.194902397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195468264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19558015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[0]R flush_numbers[0..12]: [63, 60, 53, 61, 58, 52, 58, 63, 52, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195644676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[0]R generation[0..12]: [38, 36, 32, 36, 34, 30, 34, 38, 30, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19567305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195688696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[1]R flush_numbers[0..12]: [63, 60, 53, 61, 58, 52, 58, 63, 52, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195702913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[1]R generation[0..12]: [38, 36, 32, 36, 34, 30, 34, 38, 30, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19571629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195730067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[2]R flush_numbers[0..12]: [63, 60, 53, 61, 58, 52, 58, 63, 52, 63, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195743554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[2]R generation[0..12]: [38, 36, 32, 36, 34, 30, 34, 38, 30, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195756702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195770619Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 39","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195795213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Generation requested: 40 >= found:39","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19580868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195821517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195834135Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"":"downstairs","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195846992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.19585872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"0","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195871827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"1","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195884644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"client":"2","":"downstairs","session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195897581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195909598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"21ddce00-1970-4307-bbab-55cc026cf6b3 is now active with session: a169a835-1710-4949-a0d5-3cef251ad182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195922116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195933623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344,"session_id":"a169a835-1710-4949-a0d5-3cef251ad182"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:19.195945481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 8192 data: 4 4 02/30 Write block 202 len 28672 data: 5 5 5 6 5 7 7 03/30 Write block 446 len 4096 data: 5 04/30 Write block 188 len 32768 data: 7 6 7 6 7 5 4 4 05/30 Write block 289 len 40960 data: 4 3 4 3 3 5 5 5 4 3 06/30 Read block 588 len 28672 07/30 Read block 285 len 4096 08/30 Read block 344 len 8192 09/30 Read block 586 len 16384 10/30 Flush 11/30 Write block 80 len 24576 data: 4 4 6 6 6 6 12/30 Flush 13/30 Write block 588 len 24576 data: 3 2 2 3 4 4 14/30 Write block 316 len 40960 data: 5 4 4 4 4 4 3 4 4 3 15/30 Write block 87 len 36864 data: 4 3 3 3 3 3 3 2 2 16/30 Write block 10 len 16384 data: 6 5 6 7 17/30 Read block 573 len 4096 18/30 Write block 525 len 20480 data: 6 5 4 5 7 19/30 Read block 130 len 32768 20/30 Write block 262 len 16384 data: 4 4 5 5 21/30 Read block 198 len 24576 22/30 Read block 519 len 32768 23/30 Read block 268 len 8192 24/30 Read block 563 len 16384 25/30 Write block 370 len 16384 data: 3 3 3 4 26/30 Read block 420 len 24576 27/30 Write block 566 len 32768 data: 4 5 5 5 4 4 5 5 28/30 Read block 58 len 4096 29/30 Write block 445 len 40960 data: 6 6 4 5 4 5 5 4 5 4 30/30 Read block 38 len 28672 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 10 Sent Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 16 16 18 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:19.21588889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1344} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 T T T 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 40 40 40 65 65 65 T T T 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 40 40 40 64 64 64 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 38 38 38 63 63 63 T T T 14 280-299 40 40 40 64 64 64 F F F 15 300-319 34 34 34 58 58 58 T T T 16 320-339 38 38 38 63 63 63 T T T 17 340-359 37 37 37 62 62 62 F F F 18 360-379 32 32 32 53 53 53 T T T 19 380-399 38 38 38 63 63 63 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 40 40 40 64 64 64 T T T 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 34 34 34 58 58 58 T T T 27 540-559 34 34 34 58 58 58 F F F 28 560-579 38 38 38 63 63 63 T T T 29 580-599 38 38 38 63 63 63 T T T Max gen: 40, Max flush: 65 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 40 40 40 64 64 64 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 41 41 41 66 66 66 F F F 14 280-299 40 40 40 64 64 64 F F F 15 300-319 41 41 41 66 66 66 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 37 37 37 62 62 62 F F F 18 360-379 41 41 41 66 66 66 F F F 19 380-399 38 38 38 63 63 63 F F F 20 400-419 35 35 35 59 59 59 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 41 41 41 66 66 66 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 61 61 61 F F F 26 520-539 41 41 41 66 66 66 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 41 41 41 66 66 66 F F F 29 580-599 41 41 41 66 66 66 F F F Max gen: 41, Max flush: 66 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.516228291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:23.557962281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.558370672Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.58912644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.589747185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.589780748Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.589794245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.589813061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} {"msg":"Crucible 744f4dfa-7d18-4813-8ed7-bfa7c814d8c4 has session id: 5ac76fe6-8f74-451c-a668-db6014626998","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.590447433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Upstairs opts: Upstairs UUID: 744f4dfa-7d18-4813-8ed7-bfa7c814d8c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.590494823Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Crucible stats registered with UUID: 744f4dfa-7d18-4813-8ed7-bfa7c814d8c4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.59051187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:23.590524557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.590898036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.590920411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} {"msg":"744f4dfa-7d18-4813-8ed7-bfa7c814d8c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.590986287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591005952Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591041705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591067859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591095803Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591122407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591149092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591179785Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591208049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591235503Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"ds_connection connected from Ok(127.0.0.1:59963)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591261917Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"ds_connection connected from Ok(127.0.0.1:45082)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591285682Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"ds_connection connected from Ok(127.0.0.1:49141)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591310917Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591889751Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.591918185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.592082819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.592571013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.592610705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.592639139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.59318603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593212055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[0]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593226371Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[0]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593244178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593258855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[1]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593272672Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[1]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593286219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593299816Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[2]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593312763Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[2]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.59332577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593339167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593351804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593364841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593379349Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593407222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"":"downstairs","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593421119Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593432877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"0","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593448034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"1","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593460901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"client":"2","":"downstairs","session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593473618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593485675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"744f4dfa-7d18-4813-8ed7-bfa7c814d8c4 is now active with session: 5ac76fe6-8f74-451c-a668-db6014626998","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593498373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.5935101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352,"session_id":"5ac76fe6-8f74-451c-a668-db6014626998"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:23.593522067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 526 len 36864 data: 6 5 6 8 7 8 7 7 6 03/30 Write block 386 len 12288 data: 2 2 2 04/30 Write block 486 len 12288 data: 3 2 3 05/30 Flush 06/30 Flush 07/30 Write block 352 len 24576 data: 3 3 3 3 3 2 08/30 Read block 397 len 40960 09/30 Write block 188 len 4096 data: 8 10/30 Write block 284 len 4096 data: 4 11/30 Write block 380 len 4096 data: 4 12/30 Read block 291 len 8192 13/30 Write block 505 len 32768 data: 4 4 4 4 4 4 3 3 14/30 Read block 51 len 32768 15/30 Read block 340 len 16384 16/30 Read block 546 len 36864 17/30 Write block 379 len 36864 data: 4 5 3 3 3 4 3 3 3 18/30 Read block 326 len 12288 19/30 Read block 404 len 16384 20/30 Write block 397 len 32768 data: 3 4 4 4 6 5 4 4 21/30 Write block 302 len 12288 data: 3 4 4 22/30 Read block 262 len 8192 23/30 Read block 42 len 36864 24/30 Read block 355 len 32768 25/30 Write block 448 len 32768 data: 6 5 6 6 5 6 5 4 26/30 Write block 492 len 4096 data: 3 27/30 Write block 235 len 4096 data: 4 28/30 Write block 192 len 4096 data: 8 29/30 Write block 301 len 20480 data: 3 4 5 5 4 30/30 Read block 390 len 32768 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:23.611475668Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1352} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 40 40 40 64 64 64 T T T 10 200-219 40 40 40 64 64 64 F F F 11 220-239 40 40 40 64 64 64 T T T 12 240-259 34 34 34 58 58 58 F F F 13 260-279 41 41 41 66 66 66 F F F 14 280-299 40 40 40 64 64 64 T T T 15 300-319 41 41 41 66 66 66 T T T 16 320-339 41 41 41 66 66 66 F F F 17 340-359 37 37 37 62 62 62 T T T 18 360-379 41 41 41 66 66 66 T T T 19 380-399 42 42 42 68 68 68 T T T 20 400-419 35 35 35 59 59 59 T T T 21 420-439 38 38 38 63 63 63 F F F 22 440-459 41 41 41 66 66 66 T T T 23 460-479 30 30 30 52 52 52 F F F 24 480-499 42 42 42 68 68 68 T T T 25 500-519 36 36 36 61 61 61 T T T 26 520-539 42 42 42 68 68 68 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 41 41 41 66 66 66 F F F 29 580-599 41 41 41 66 66 66 F F F Max gen: 42, Max flush: 68 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 43 43 43 69 69 69 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 43 43 43 69 69 69 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 41 41 41 66 66 66 F F F 14 280-299 43 43 43 69 69 69 F F F 15 300-319 43 43 43 69 69 69 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 43 43 43 69 69 69 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 43 43 43 69 69 69 F F F 20 400-419 43 43 43 69 69 69 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 43 43 43 69 69 69 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 43 43 43 69 69 69 F F F 25 500-519 43 43 43 69 69 69 F F F 26 520-539 42 42 42 68 68 68 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 41 41 41 66 66 66 F F F 29 580-599 41 41 41 66 66 66 F F F Max gen: 43, Max flush: 69 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.904222613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:27.946240271Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.946672708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.977480914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.97809706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.978131433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.9781449Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.978159097Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} {"msg":"Crucible 00cca7e3-c8c2-4f92-898c-24e4342e0400 has session id: 94f8cb3a-ec51-466b-930e-79cb53c9da32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.978846578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Upstairs opts: Upstairs UUID: 00cca7e3-c8c2-4f92-898c-24e4342e0400, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.97888181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Crucible stats registered with UUID: 00cca7e3-c8c2-4f92-898c-24e4342e0400","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.978897557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:27.978915733Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979354838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979378683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} {"msg":"00cca7e3-c8c2-4f92-898c-24e4342e0400 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979454066Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979472613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979495288Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979508845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979522831Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979535799Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979549736Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979562623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.97957602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979589797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"ds_connection connected from Ok(127.0.0.1:33231)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.979740594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"ds_connection connected from Ok(127.0.0.1:34692)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.980028992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"ds_connection connected from Ok(127.0.0.1:40619)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.980067344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.980516616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.980543541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.980558747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981184562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981340958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.98137848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981768455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981910754Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[0]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 69, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981971111Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[0]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 43, 40, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.981991007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982005734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[1]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 69, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982019561Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[1]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 43, 40, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982033198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982047105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[2]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 58, 63, 52, 69, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982064451Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[2]R generation[0..12]: [41, 36, 32, 36, 41, 30, 34, 38, 30, 43, 40, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982078038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982091845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:39:27.982104362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982120279Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982132836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982145463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"":"downstairs","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982158331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982169998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"0","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982182955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"1","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982195512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"client":"2","":"downstairs","session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.98220827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982220267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"00cca7e3-c8c2-4f92-898c-24e4342e0400 is now active with session: 94f8cb3a-ec51-466b-930e-79cb53c9da32","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982232715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2025-01-23T17:39:27.982247661Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360,"session_id":"94f8cb3a-ec51-466b-930e-79cb53c9da32"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:27.982261428Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 28672 data: 5 4 4 4 4 4 3 02/30 Write block 418 len 12288 data: 5 4 4 03/30 Write block 121 len 16384 data: 6 5 3 3 04/30 Write block 380 len 16384 data: 6 4 4 4 05/30 Write block 123 len 12288 data: 4 4 3 06/30 Write block 183 len 40960 data: 6 7 6 7 8 9 7 8 7 9 07/30 Read block 315 len 4096 08/30 Write block 479 len 12288 data: 3 5 6 09/30 Read block 376 len 20480 10/30 Write block 552 len 36864 data: 3 3 3 3 4 4 4 5 5 11/30 Read block 265 len 8192 12/30 Write block 588 len 4096 data: 4 13/30 Write block 174 len 8192 data: 5 6 14/30 Write block 173 len 28672 data: 5 6 7 5 5 6 4 15/30 Write block 251 len 40960 data: 3 2 3 3 4 5 5 5 5 4 16/30 Write block 550 len 36864 data: 5 3 4 4 4 4 5 5 5 17/30 Flush 18/30 Write block 159 len 24576 data: 6 5 5 5 5 5 19/30 Read block 165 len 8192 20/30 Read block 172 len 28672 21/30 Read block 426 len 40960 22/30 Read block 237 len 12288 23/30 Read block 37 len 24576 24/30 Write block 206 len 4096 data: 6 25/30 Read block 80 len 20480 26/30 Flush 27/30 Write block 427 len 24576 data: 3 3 3 3 3 3 28/30 Write block 186 len 4096 data: 8 29/30 Write block 456 len 28672 data: 4 3 3 2 2 2 3 30/30 Write block 508 len 12288 data: 5 5 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:28.000095975Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1360} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 44 44 44 70 70 70 T T T 10 200-219 44 44 44 71 71 71 F F F 11 220-239 43 43 43 69 69 69 F F F 12 240-259 44 44 44 70 70 70 F F F 13 260-279 44 44 44 70 70 70 F F F 14 280-299 43 43 43 69 69 69 F F F 15 300-319 43 43 43 69 69 69 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 43 43 43 69 69 69 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 44 44 44 70 70 70 F F F 20 400-419 44 44 44 70 70 70 F F F 21 420-439 44 44 44 70 70 70 T T T 22 440-459 43 43 43 69 69 69 T T T 23 460-479 44 44 44 70 70 70 T T T 24 480-499 44 44 44 70 70 70 F F F 25 500-519 43 43 43 69 69 69 T T T 26 520-539 42 42 42 68 68 68 F F F 27 540-559 44 44 44 70 70 70 F F F 28 560-579 44 44 44 70 70 70 F F F 29 580-599 44 44 44 70 70 70 F F F Max gen: 44, Max flush: 71 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 36 36 36 60 60 60 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 44 44 44 71 71 71 F F F 9 180-199 45 45 45 72 72 72 F F F 10 200-219 44 44 44 71 71 71 F F F 11 220-239 43 43 43 69 69 69 F F F 12 240-259 44 44 44 70 70 70 F F F 13 260-279 44 44 44 70 70 70 F F F 14 280-299 43 43 43 69 69 69 F F F 15 300-319 43 43 43 69 69 69 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 43 43 43 69 69 69 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 44 44 44 70 70 70 F F F 20 400-419 44 44 44 70 70 70 F F F 21 420-439 45 45 45 72 72 72 F F F 22 440-459 45 45 45 72 72 72 F F F 23 460-479 45 45 45 72 72 72 F F F 24 480-499 44 44 44 70 70 70 F F F 25 500-519 45 45 45 72 72 72 F F F 26 520-539 42 42 42 68 68 68 F F F 27 540-559 44 44 44 70 70 70 F F F 28 560-579 44 44 44 70 70 70 F F F 29 580-599 44 44 44 70 70 70 F F F Max gen: 45, Max flush: 72 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.269746114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:32.312201467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.312655458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.345195289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.345879351Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.345914494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.345933219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.345948366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} {"msg":"Crucible bdea9d77-716c-4954-9c4b-83ee8ab39c9e has session id: 8e79559c-ddd5-49bf-94a1-9ef9f682f09f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.346550106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Upstairs opts: Upstairs UUID: bdea9d77-716c-4954-9c4b-83ee8ab39c9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.346596686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Crucible stats registered with UUID: bdea9d77-716c-4954-9c4b-83ee8ab39c9e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.346627859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:32.346658513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347072203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347104586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} {"msg":"bdea9d77-716c-4954-9c4b-83ee8ab39c9e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347156465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347183309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347220421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347248035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347275079Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347299414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347326918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347354062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347378477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347430245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42080)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347461339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48643)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347488782Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48461)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.347514687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.348183952Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.348225193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.348257126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.348765966Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.34879535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.348811286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349310528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349410626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[0]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 70, 71, 71, 72, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349471833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[0]R generation[0..12]: [41, 36, 32, 36, 41, 30, 44, 44, 44, 45, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349494638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349510544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[1]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 70, 71, 71, 72, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349524781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[1]R generation[0..12]: [41, 36, 32, 36, 41, 30, 44, 44, 44, 45, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349538538Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349552275Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[2]R flush_numbers[0..12]: [66, 60, 53, 61, 66, 52, 70, 71, 71, 72, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349565453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[2]R generation[0..12]: [41, 36, 32, 36, 41, 30, 44, 44, 44, 45, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.34957851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349592177Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349610623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.34962324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349635648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349648125Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"":"downstairs","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349660702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349672209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"0","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349689126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"1","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349702203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"client":"2","":"downstairs","session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.34971477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349727628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"bdea9d77-716c-4954-9c4b-83ee8ab39c9e is now active with session: 8e79559c-ddd5-49bf-94a1-9ef9f682f09f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349739975Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.349751582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368,"session_id":"8e79559c-ddd5-49bf-94a1-9ef9f682f09f"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:32.34976341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 497 len 40960 data: 4 4 3 5 4 5 5 4 5 5 02/30 Read block 3 len 16384 03/30 Write block 181 len 20480 data: 4 5 7 8 7 04/30 Read block 341 len 8192 05/30 Read block 227 len 12288 06/30 Read block 63 len 16384 07/30 Read block 595 len 16384 08/30 Write block 586 len 16384 data: 3 3 5 3 09/30 Flush 10/30 Write block 110 len 16384 data: 4 4 4 3 11/30 Write block 356 len 12288 data: 4 3 2 12/30 Write block 429 len 20480 data: 4 4 4 4 3 13/30 Write block 301 len 32768 data: 4 5 6 6 5 4 3 4 14/30 Write block 504 len 12288 data: 5 6 6 15/30 Read block 164 len 4096 16/30 Write block 24 len 36864 data: 4 5 5 7 7 4 4 3 4 17/30 Flush 18/30 Write block 274 len 20480 data: 5 5 5 5 5 19/30 Read block 275 len 8192 20/30 Write block 179 len 16384 data: 5 5 5 6 21/30 Write block 466 len 32768 data: 4 5 5 4 5 5 5 2 22/30 Read block 148 len 12288 23/30 Read block 390 len 4096 24/30 Read block 443 len 12288 25/30 Read block 531 len 32768 26/30 Write block 539 len 32768 data: 10 11 11 10 9 9 7 6 27/30 Write block 582 len 4096 data: 5 28/30 Write block 489 len 12288 data: 3 3 3 29/30 Write block 23 len 8192 data: 5 5 30/30 Write block 156 len 16384 data: 4 4 4 7 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:32.368325419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1368} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 46 46 46 74 74 74 T T T 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 46 46 46 74 74 74 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 44 44 44 71 71 71 T T T 8 160-179 44 44 44 71 71 71 T T T 9 180-199 46 46 46 73 73 73 T T T 10 200-219 44 44 44 71 71 71 F F F 11 220-239 43 43 43 69 69 69 F F F 12 240-259 44 44 44 70 70 70 F F F 13 260-279 44 44 44 70 70 70 T T T 14 280-299 43 43 43 69 69 69 F F F 15 300-319 46 46 46 74 74 74 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 44 44 44 70 70 70 F F F 20 400-419 44 44 44 70 70 70 F F F 21 420-439 46 46 46 74 74 74 F F F 22 440-459 45 45 45 72 72 72 F F F 23 460-479 45 45 45 72 72 72 T T T 24 480-499 46 46 46 73 73 73 T T T 25 500-519 46 46 46 74 74 74 F F F 26 520-539 42 42 42 68 68 68 T T T 27 540-559 44 44 44 70 70 70 T T T 28 560-579 44 44 44 70 70 70 F F F 29 580-599 46 46 46 73 73 73 T T T Max gen: 46, Max flush: 74 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 F F F 1 020-039 47 47 47 75 75 75 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 46 46 46 74 74 74 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 47 47 47 75 75 75 F F F 8 160-179 47 47 47 75 75 75 F F F 9 180-199 47 47 47 75 75 75 F F F 10 200-219 44 44 44 71 71 71 F F F 11 220-239 43 43 43 69 69 69 F F F 12 240-259 44 44 44 70 70 70 F F F 13 260-279 47 47 47 75 75 75 F F F 14 280-299 43 43 43 69 69 69 F F F 15 300-319 46 46 46 74 74 74 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 44 44 44 70 70 70 F F F 20 400-419 44 44 44 70 70 70 F F F 21 420-439 46 46 46 74 74 74 F F F 22 440-459 45 45 45 72 72 72 F F F 23 460-479 47 47 47 75 75 75 F F F 24 480-499 47 47 47 75 75 75 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 47 47 47 75 75 75 F F F 27 540-559 47 47 47 75 75 75 F F F 28 560-579 44 44 44 70 70 70 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 47, Max flush: 75 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.658879638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.702577762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.703300435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.703343646Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.703358743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.70337326Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} {"msg":"Crucible 199da9ec-ad53-4e1e-b7a0-f8f7694a30b9 has session id: e0099ab5-e70d-4e08-8efc-eef8c5697ae5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.703985427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Upstairs opts: Upstairs UUID: 199da9ec-ad53-4e1e-b7a0-f8f7694a30b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704028798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Crucible stats registered with UUID: 199da9ec-ad53-4e1e-b7a0-f8f7694a30b9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704046134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:36.704059341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704630277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704654662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} {"msg":"199da9ec-ad53-4e1e-b7a0-f8f7694a30b9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704715809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704733515Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704760879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704775266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704799141Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704819826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.70484613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704870575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.70489709Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704924034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51316)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.704969054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39766)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.705002157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54303)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.705135388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.705754954Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.705782128Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.705797844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.706405613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.706433806Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.706449853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.706980578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707108471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[0]R flush_numbers[0..12]: [66, 75, 53, 61, 66, 74, 70, 75, 75, 75, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707164048Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[0]R generation[0..12]: [41, 47, 32, 36, 41, 46, 44, 47, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707182774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707197201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[1]R flush_numbers[0..12]: [66, 75, 53, 61, 66, 74, 70, 75, 75, 75, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707210698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[1]R generation[0..12]: [41, 47, 32, 36, 41, 46, 44, 47, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707223935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707237583Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[2]R flush_numbers[0..12]: [66, 75, 53, 61, 66, 74, 70, 75, 75, 75, 71, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.70725114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[2]R generation[0..12]: [41, 47, 32, 36, 41, 46, 44, 47, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707264587Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707278044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.70729494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Generation requested: 48 >= found:48",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707314256Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707334302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707357586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"":"downstairs","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707372363Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707392579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"0","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707412794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"1","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707426411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"client":"2","":"downstairs","session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707439149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707452756Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"199da9ec-ad53-4e1e-b7a0-f8f7694a30b9 is now active with session: e0099ab5-e70d-4e08-8efc-eef8c5697ae5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707475081Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :30,"time":"2025-01-23T17:39:36.707495137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376,"session_id":"e0099ab5-e70d-4e08-8efc-eef8c5697ae5"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:36.707507734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 28672 data: 4 5 4 4 5 5 5 02/30 Write block 26 len 28672 data: 6 8 8 5 5 4 5 03/30 Flush 04/30 Write block 15 len 40960 data: 5 5 5 4 4 4 4 5 6 6 05/30 Read block 467 len 4096 06/30 Read block 273 len 32768 07/30 Read block 472 len 4096 08/30 Write block 155 len 20480 data: 2 5 5 5 8 09/30 Write block 228 len 8192 data: 3 3 10/30 Read block 104 len 32768 11/30 Read block 106 len 32768 12/30 Write block 561 len 8192 data: 4 3 13/30 Read block 139 len 28672 14/30 Write block 424 len 24576 data: 4 4 3 4 4 5 15/30 Write block 311 len 12288 data: 5 5 6 16/30 Write block 229 len 16384 data: 4 3 4 4 17/30 Read block 455 len 4096 18/30 Write block 387 len 32768 data: 4 3 2 3 4 4 4 5 19/30 Write block 471 len 16384 data: 6 6 3 2 20/30 Write block 211 len 16384 data: 5 4 4 4 21/30 Write block 156 len 28672 data: 6 6 6 9 6 6 6 22/30 Write block 285 len 32768 data: 5 4 4 4 5 4 5 4 23/30 Write block 569 len 4096 data: 6 24/30 Read block 339 len 12288 25/30 Read block 0 len 8192 26/30 Read block 444 len 12288 27/30 Write block 454 len 4096 data: 6 28/30 Read block 29 len 20480 29/30 Read block 205 len 8192 30/30 Read block 24 len 24576 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 26 27 27 80 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:36.724624326Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1376} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 66 66 66 T T T 1 020-039 48 48 48 76 76 76 T T T 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 46 46 46 74 74 74 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 47 47 47 75 75 75 T T T 8 160-179 47 47 47 75 75 75 T T T 9 180-199 47 47 47 75 75 75 F F F 10 200-219 44 44 44 71 71 71 T T T 11 220-239 43 43 43 69 69 69 T T T 12 240-259 44 44 44 70 70 70 F F F 13 260-279 47 47 47 75 75 75 F F F 14 280-299 43 43 43 69 69 69 T T T 15 300-319 46 46 46 74 74 74 T T T 16 320-339 41 41 41 66 66 66 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 48 48 48 76 76 76 T T T 20 400-419 48 48 48 76 76 76 F F F 21 420-439 46 46 46 74 74 74 T T T 22 440-459 45 45 45 72 72 72 T T T 23 460-479 47 47 47 75 75 75 T T T 24 480-499 47 47 47 75 75 75 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 47 47 47 75 75 75 F F F 27 540-559 47 47 47 75 75 75 F F F 28 560-579 44 44 44 70 70 70 T T T 29 580-599 47 47 47 75 75 75 F F F Max gen: 48, Max flush: 76 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 77 77 77 F F F 1 020-039 49 49 49 77 77 77 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 F F F 5 100-119 46 46 46 74 74 74 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 49 49 49 77 77 77 F F F 8 160-179 49 49 49 77 77 77 F F F 9 180-199 47 47 47 75 75 75 F F F 10 200-219 49 49 49 77 77 77 F F F 11 220-239 49 49 49 77 77 77 F F F 12 240-259 44 44 44 70 70 70 F F F 13 260-279 47 47 47 75 75 75 F F F 14 280-299 49 49 49 77 77 77 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 41 41 41 66 66 66 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 49 49 49 77 77 77 F F F 20 400-419 48 48 48 76 76 76 F F F 21 420-439 49 49 49 77 77 77 F F F 22 440-459 49 49 49 77 77 77 F F F 23 460-479 49 49 49 77 77 77 F F F 24 480-499 47 47 47 75 75 75 F F F 25 500-519 46 46 46 74 74 74 F F F 26 520-539 47 47 47 75 75 75 F F F 27 540-559 47 47 47 75 75 75 F F F 28 560-579 49 49 49 77 77 77 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 49, Max flush: 77 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.013698537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:41.055941146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.056461053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.087278237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088175213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088210375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088230111Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088244898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} {"msg":"Crucible e2ff275a-3818-48b0-87c7-8fee4fe374be has session id: a5c04159-2c35-48d4-9ef1-647b08fe567e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088825892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Upstairs opts: Upstairs UUID: e2ff275a-3818-48b0-87c7-8fee4fe374be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088889048Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Crucible stats registered with UUID: e2ff275a-3818-48b0-87c7-8fee4fe374be","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.088916432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:41.088938438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089318905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.08934302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} {"msg":"e2ff275a-3818-48b0-87c7-8fee4fe374be active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089407176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089424662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089447228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089460994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089475331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089488508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089509334Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089533909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089556844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089581019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55578)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089604813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62726)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.089629138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36833)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.08980476Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.090396522Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.090433894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.090556397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091024006Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091198358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091293517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091668966Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091738831Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[0]R flush_numbers[0..12]: [77, 77, 53, 61, 66, 74, 70, 77, 77, 75, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091869373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[0]R generation[0..12]: [49, 49, 32, 36, 41, 46, 44, 49, 49, 47, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091890868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091905945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[1]R flush_numbers[0..12]: [77, 77, 53, 61, 66, 74, 70, 77, 77, 75, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091920582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[1]R generation[0..12]: [49, 49, 32, 36, 41, 46, 44, 49, 49, 47, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091944716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091972121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[2]R flush_numbers[0..12]: [77, 77, 53, 61, 66, 74, 70, 77, 77, 75, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.091996865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[2]R generation[0..12]: [49, 49, 32, 36, 41, 46, 44, 49, 49, 47, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092017661Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092036607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:39:41.092049764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.09206507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092077688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092090225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"":"downstairs","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092103282Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092119129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"0","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092132806Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"1","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092145763Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"client":"2","":"downstairs","session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.09215879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092170827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"e2ff275a-3818-48b0-87c7-8fee4fe374be is now active with session: a5c04159-2c35-48d4-9ef1-647b08fe567e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092183325Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.092194862Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384,"session_id":"a5c04159-2c35-48d4-9ef1-647b08fe567e"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:41.09220688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 20480 data: 4 3 4 6 6 02/30 Read block 294 len 12288 03/30 Write block 414 len 36864 data: 6 5 4 5 6 5 5 4 3 04/30 Flush 05/30 Write block 293 len 8192 data: 4 6 06/30 Read block 219 len 16384 07/30 Read block 38 len 8192 08/30 Read block 192 len 32768 09/30 Read block 29 len 32768 10/30 Flush 11/30 Write block 53 len 8192 data: 4 5 12/30 Read block 181 len 28672 13/30 Write block 289 len 16384 data: 6 5 6 5 14/30 Read block 342 len 36864 15/30 Read block 303 len 20480 16/30 Read block 157 len 16384 17/30 Read block 224 len 40960 18/30 Read block 121 len 28672 19/30 Write block 226 len 32768 data: 4 3 4 5 4 5 5 4 20/30 Read block 352 len 4096 21/30 Read block 384 len 28672 22/30 Write block 166 len 4096 data: 3 23/30 Write block 496 len 32768 data: 5 5 5 4 6 5 6 6 24/30 Read block 226 len 16384 25/30 Read block 424 len 28672 26/30 Read block 268 len 4096 27/30 Write block 206 len 12288 data: 7 8 8 28/30 Write block 97 len 40960 data: 3 4 3 4 4 5 5 5 4 4 29/30 Write block 426 len 8192 data: 4 5 30/30 Write block 232 len 36864 data: 6 5 4 5 3 3 4 4 5 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 2 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:41.112495895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1384} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 77 77 77 F F F 1 020-039 49 49 49 77 77 77 F F F 2 040-059 32 32 32 53 53 53 T T T 3 060-079 36 36 36 61 61 61 F F F 4 080-099 41 41 41 66 66 66 T T T 5 100-119 46 46 46 74 74 74 T T T 6 120-139 44 44 44 70 70 70 F F F 7 140-159 49 49 49 77 77 77 F F F 8 160-179 49 49 49 77 77 77 T T T 9 180-199 47 47 47 75 75 75 F F F 10 200-219 49 49 49 77 77 77 T T T 11 220-239 49 49 49 77 77 77 T T T 12 240-259 44 44 44 70 70 70 T T T 13 260-279 47 47 47 75 75 75 F F F 14 280-299 50 50 50 79 79 79 T T T 15 300-319 49 49 49 77 77 77 F F F 16 320-339 50 50 50 78 78 78 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 49 49 49 77 77 77 F F F 20 400-419 50 50 50 78 78 78 F F F 21 420-439 50 50 50 78 78 78 T T T 22 440-459 49 49 49 77 77 77 F F F 23 460-479 49 49 49 77 77 77 F F F 24 480-499 47 47 47 75 75 75 T T T 25 500-519 46 46 46 74 74 74 T T T 26 520-539 47 47 47 75 75 75 F F F 27 540-559 47 47 47 75 75 75 F F F 28 560-579 49 49 49 77 77 77 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 50, Max flush: 79 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 77 77 77 F F F 1 020-039 49 49 49 77 77 77 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 51 51 51 80 80 80 F F F 5 100-119 51 51 51 80 80 80 F F F 6 120-139 44 44 44 70 70 70 F F F 7 140-159 49 49 49 77 77 77 F F F 8 160-179 51 51 51 80 80 80 F F F 9 180-199 47 47 47 75 75 75 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 51 51 51 80 80 80 F F F 12 240-259 51 51 51 80 80 80 F F F 13 260-279 47 47 47 75 75 75 F F F 14 280-299 51 51 51 80 80 80 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 50 50 50 78 78 78 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 49 49 49 77 77 77 F F F 20 400-419 50 50 50 78 78 78 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 49 49 49 77 77 77 F F F 23 460-479 49 49 49 77 77 77 F F F 24 480-499 51 51 51 80 80 80 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 47 47 47 75 75 75 F F F 27 540-559 47 47 47 75 75 75 F F F 28 560-579 49 49 49 77 77 77 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 51, Max flush: 80 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.421607624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.464845257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.465466213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.465508704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.46552452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.465539367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} {"msg":"Crucible 0bd4a14b-27b7-4271-bf3a-b9bddff4a3d1 has session id: ef3d1676-6321-4cba-b65e-ca11171ce493","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466086529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Upstairs opts: Upstairs UUID: 0bd4a14b-27b7-4271-bf3a-b9bddff4a3d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466136448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Crucible stats registered with UUID: 0bd4a14b-27b7-4271-bf3a-b9bddff4a3d1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466160872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:45.466176269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466595818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.46663523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} {"msg":"0bd4a14b-27b7-4271-bf3a-b9bddff4a3d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466694287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466712533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466734088Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466747955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.46677325Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466787167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466848274Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466887955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466913129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466937274Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"ds_connection connected from Ok(127.0.0.1:43840)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466962199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"ds_connection connected from Ok(127.0.0.1:53456)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.466986254Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:45.467013688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:45.467041902Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:45.467073275Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.467251686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.467609049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.467762236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.468162679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.468304518Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.468738204Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:45.468817567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.468927086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"ds_connection connected from Ok(127.0.0.1:36852)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.469155877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.470122077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.470654052Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[0]R flush_numbers[0..12]: [77, 77, 80, 61, 80, 80, 70, 77, 80, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471246234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[0]R generation[0..12]: [49, 49, 51, 36, 51, 51, 44, 49, 51, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471276867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471303411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[1]R flush_numbers[0..12]: [77, 77, 80, 61, 80, 80, 70, 77, 80, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471322197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[1]R generation[0..12]: [49, 49, 51, 36, 51, 51, 44, 49, 51, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471340183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471358359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[2]R flush_numbers[0..12]: [77, 77, 80, 61, 80, 80, 70, 77, 80, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471376066Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[2]R generation[0..12]: [49, 49, 51, 36, 51, 51, 44, 49, 51, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471393562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471411748Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:39:55.471428394Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471452139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471468426Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471485192Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"":"downstairs","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471502458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471517955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"0","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471535001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"1","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471555847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"client":"2","":"downstairs","session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471574363Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471590599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"0bd4a14b-27b7-4271-bf3a-b9bddff4a3d1 is now active with session: ef3d1676-6321-4cba-b65e-ca11171ce493","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471608585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471624312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392,"session_id":"ef3d1676-6321-4cba-b65e-ca11171ce493"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:39:55.471639918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 134 len 8192 data: 7 7 02/30 Write block 396 len 32768 data: 5 5 6 6 6 7 6 5 03/30 Read block 528 len 32768 04/30 Write block 142 len 36864 data: 5 5 4 5 6 6 5 6 6 05/30 Read block 555 len 12288 06/30 Read block 244 len 32768 07/30 Read block 233 len 20480 08/30 Read block 269 len 32768 09/30 Write block 178 len 20480 data: 7 6 6 6 7 10/30 Read block 15 len 40960 11/30 Write block 175 len 16384 data: 8 6 6 8 12/30 Write block 339 len 16384 data: 4 7 5 8 13/30 Read block 124 len 20480 14/30 Read block 446 len 36864 15/30 Write block 19 len 8192 data: 5 5 16/30 Write block 539 len 24576 data: 11 12 12 11 10 10 17/30 Write block 278 len 28672 data: 6 5 5 5 5 5 5 18/30 Read block 207 len 16384 19/30 Read block 531 len 20480 20/30 Read block 407 len 16384 21/30 Read block 502 len 40960 22/30 Read block 265 len 12288 23/30 Read block 578 len 36864 24/30 Read block 559 len 20480 25/30 Write block 533 len 16384 data: 8 7 5 3 26/30 Write block 467 len 12288 data: 6 6 5 27/30 Write block 80 len 40960 data: 5 5 7 7 7 7 5 5 4 4 28/30 Write block 459 len 40960 data: 3 3 3 4 3 3 4 5 7 7 29/30 Read block 333 len 28672 30/30 Read block 195 len 40960 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:39:55.494033318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1392} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 77 77 77 T T T 1 020-039 49 49 49 77 77 77 T T T 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 51 51 51 80 80 80 T T T 5 100-119 51 51 51 80 80 80 F F F 6 120-139 44 44 44 70 70 70 T T T 7 140-159 49 49 49 77 77 77 T T T 8 160-179 51 51 51 80 80 80 T T T 9 180-199 47 47 47 75 75 75 T T T 10 200-219 51 51 51 80 80 80 F F F 11 220-239 51 51 51 80 80 80 F F F 12 240-259 51 51 51 80 80 80 F F F 13 260-279 47 47 47 75 75 75 T T T 14 280-299 51 51 51 80 80 80 T T T 15 300-319 49 49 49 77 77 77 F F F 16 320-339 50 50 50 78 78 78 T T T 17 340-359 46 46 46 74 74 74 T T T 18 360-379 43 43 43 69 69 69 F F F 19 380-399 49 49 49 77 77 77 T T T 20 400-419 50 50 50 78 78 78 T T T 21 420-439 51 51 51 80 80 80 F F F 22 440-459 49 49 49 77 77 77 T T T 23 460-479 49 49 49 77 77 77 T T T 24 480-499 51 51 51 80 80 80 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 47 47 47 75 75 75 T T T 27 540-559 47 47 47 75 75 75 T T T 28 560-579 49 49 49 77 77 77 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 51, Max flush: 80 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 81 81 81 F F F 1 020-039 52 52 52 81 81 81 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 52 52 52 81 81 81 F F F 5 100-119 51 51 51 80 80 80 F F F 6 120-139 52 52 52 81 81 81 F F F 7 140-159 52 52 52 81 81 81 F F F 8 160-179 52 52 52 81 81 81 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 51 51 51 80 80 80 F F F 12 240-259 51 51 51 80 80 80 F F F 13 260-279 52 52 52 81 81 81 F F F 14 280-299 52 52 52 81 81 81 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 52 52 52 81 81 81 F F F 17 340-359 52 52 52 81 81 81 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 52 52 52 81 81 81 F F F 20 400-419 52 52 52 81 81 81 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 52 52 52 81 81 81 F F F 23 460-479 52 52 52 81 81 81 F F F 24 480-499 51 51 51 80 80 80 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 52 52 52 81 81 81 F F F 27 540-559 52 52 52 81 81 81 F F F 28 560-579 49 49 49 77 77 77 F F F 29 580-599 47 47 47 75 75 75 F F F Max gen: 52, Max flush: 81 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.808307903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.852382215Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.853680944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.853725324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.853740471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.853755198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} {"msg":"Crucible 591aa8df-fb7e-41d7-ae26-c9259c255a8f has session id: 319b53b1-9924-4442-b8cd-d7ba8cefe007","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854385072Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Upstairs opts: Upstairs UUID: 591aa8df-fb7e-41d7-ae26-c9259c255a8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854442169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Crucible stats registered with UUID: 591aa8df-fb7e-41d7-ae26-c9259c255a8f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854471623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:09.854498157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854895421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854927154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} {"msg":"591aa8df-fb7e-41d7-ae26-c9259c255a8f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.854987281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855015015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.85503874Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855053237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855067653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.85508461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855110564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855136619Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855161273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855183848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"ds_connection connected from Ok(127.0.0.1:33163)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855208223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:09.855241826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"ds_connection connected from Ok(127.0.0.1:38055)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.8552691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:09.855298434Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:09.855353312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855598929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.855926707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.85609854Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.856604311Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.856722415Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.857318276Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:09.857352189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.857076263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"ds_connection connected from Ok(127.0.0.1:40232)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.857578234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.858551133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859067781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[0]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 80, 81, 81, 81, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859632249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[0]R generation[0..12]: [52, 52, 51, 36, 52, 51, 52, 52, 52, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859657264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859678249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[1]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 80, 81, 81, 81, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859693556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[1]R generation[0..12]: [52, 52, 51, 36, 52, 51, 52, 52, 52, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859707873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859722239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[2]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 80, 81, 81, 81, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859736076Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[2]R generation[0..12]: [52, 52, 51, 36, 52, 51, 52, 52, 52, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859749853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859764161Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859777298Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859790665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859803692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859816729Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"":"downstairs","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859830086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859852011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"0","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859865708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"1","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859878956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"client":"2","":"downstairs","session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859895042Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859907859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"591aa8df-fb7e-41d7-ae26-c9259c255a8f is now active with session: 319b53b1-9924-4442-b8cd-d7ba8cefe007","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859921096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859933284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400,"session_id":"319b53b1-9924-4442-b8cd-d7ba8cefe007"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:19.859945811Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 119 len 40960 data: 5 6 7 6 5 5 4 2 2 2 02/30 Read block 194 len 4096 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 386 len 28672 data: 4 5 4 3 4 5 5 07/30 Read block 106 len 40960 08/30 Write block 166 len 16384 data: 4 3 3 3 09/30 Read block 515 len 32768 10/30 Write block 480 len 24576 data: 6 7 5 6 7 5 11/30 Write block 100 len 16384 data: 5 5 6 6 12/30 Write block 452 len 40960 data: 6 7 7 5 5 4 4 4 4 4 13/30 Read block 546 len 4096 14/30 Read block 524 len 4096 15/30 Read block 41 len 4096 16/30 Flush 17/30 Write block 256 len 20480 data: 6 6 6 6 5 18/30 Write block 310 len 8192 data: 6 6 19/30 Write block 324 len 12288 data: 5 5 4 20/30 Read block 325 len 36864 21/30 Read block 362 len 20480 22/30 Write block 494 len 4096 data: 4 23/30 Write block 288 len 4096 data: 5 24/30 Read block 587 len 28672 25/30 Write block 385 len 24576 data: 4 5 6 5 4 5 26/30 Flush 27/30 Write block 578 len 16384 data: 4 6 5 5 28/30 Write block 133 len 16384 data: 6 8 8 7 29/30 Read block 138 len 24576 30/30 Write block 453 len 28672 data: 8 8 6 6 5 5 5 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:19.879454765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1400} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 81 81 81 F F F 1 020-039 52 52 52 81 81 81 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 52 52 52 81 81 81 F F F 5 100-119 54 54 54 85 85 85 F F F 6 120-139 54 54 54 82 82 82 T T T 7 140-159 52 52 52 81 81 81 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 51 51 51 80 80 80 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 54 54 54 86 86 86 F F F 14 280-299 54 54 54 86 86 86 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 52 52 52 81 81 81 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 54 54 54 86 86 86 F F F 20 400-419 52 52 52 81 81 81 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 54 54 54 85 85 85 T T T 23 460-479 54 54 54 85 85 85 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 52 52 52 81 81 81 F F F 27 540-559 52 52 52 81 81 81 F F F 28 560-579 49 49 49 77 77 77 T T T 29 580-599 47 47 47 75 75 75 T T T Max gen: 54, Max flush: 86 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 81 81 81 F F F 1 020-039 52 52 52 81 81 81 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 52 52 52 81 81 81 F F F 5 100-119 54 54 54 85 85 85 F F F 6 120-139 55 55 55 87 87 87 F F F 7 140-159 52 52 52 81 81 81 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 51 51 51 80 80 80 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 54 54 54 86 86 86 F F F 14 280-299 54 54 54 86 86 86 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 52 52 52 81 81 81 F F F 18 360-379 43 43 43 69 69 69 F F F 19 380-399 54 54 54 86 86 86 F F F 20 400-419 52 52 52 81 81 81 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 55 55 55 87 87 87 F F F 23 460-479 54 54 54 85 85 85 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 52 52 52 81 81 81 F F F 27 540-559 52 52 52 81 81 81 F F F 28 560-579 55 55 55 87 87 87 F F F 29 580-599 55 55 55 87 87 87 F F F Max gen: 55, Max flush: 87 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.113777797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.15918795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160142663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160195202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160219976Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160238872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} {"msg":"Crucible 9295cd68-4ebc-40a3-9f87-61c26b96d1f9 has session id: 3c96fdfa-c6e9-4c1c-8cae-21a686a635a1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160910177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Upstairs opts: Upstairs UUID: 9295cd68-4ebc-40a3-9f87-61c26b96d1f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160967284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Crucible stats registered with UUID: 9295cd68-4ebc-40a3-9f87-61c26b96d1f9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.160997848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:24.161021533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161441741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161477214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} {"msg":"9295cd68-4ebc-40a3-9f87-61c26b96d1f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16154263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161561036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16158699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161601617Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161616804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16163488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161650236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16168164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161710743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161740157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47755)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161766801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45142)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.161794035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34823)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16181852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.162570947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.162610818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.162641652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16320283Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163290871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163379492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163836103Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163867496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[0]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 85, 87, 81, 85, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16394221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[0]R generation[0..12]: [52, 52, 51, 36, 52, 54, 55, 52, 54, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163962655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163977352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[1]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 85, 87, 81, 85, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.163990949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[1]R generation[0..12]: [52, 52, 51, 36, 52, 54, 55, 52, 54, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164004266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164017863Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[2]R flush_numbers[0..12]: [81, 81, 80, 61, 81, 85, 87, 81, 85, 81, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164031121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[2]R generation[0..12]: [52, 52, 51, 36, 52, 54, 55, 52, 54, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164045218Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164058715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164071092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164083989Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164096386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164114633Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"":"downstairs","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164131189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164143256Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"0","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164156613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"1","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164169631Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"client":"2","":"downstairs","session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164182688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164194595Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"9295cd68-4ebc-40a3-9f87-61c26b96d1f9 is now active with session: 3c96fdfa-c6e9-4c1c-8cae-21a686a635a1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164207102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.16421866Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408,"session_id":"3c96fdfa-c6e9-4c1c-8cae-21a686a635a1"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:24.164231677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 36864 data: 5 5 6 5 6 5 3 2 2 02/30 Read block 489 len 28672 03/30 Write block 466 len 40960 data: 6 8 8 6 6 7 7 4 3 2 04/30 Write block 10 len 36864 data: 7 6 7 8 6 6 6 6 5 05/30 Read block 156 len 32768 06/30 Read block 244 len 4096 07/30 Read block 456 len 32768 08/30 Write block 11 len 8192 data: 7 8 09/30 Read block 167 len 12288 10/30 Write block 577 len 8192 data: 4 5 11/30 Flush 12/30 Write block 375 len 8192 data: 5 5 13/30 Read block 236 len 28672 14/30 Write block 144 len 32768 data: 5 6 7 7 6 7 7 5 15/30 Write block 98 len 20480 data: 5 4 6 6 7 16/30 Read block 487 len 24576 17/30 Write block 272 len 40960 data: 5 6 6 6 6 6 7 6 6 6 18/30 Write block 115 len 40960 data: 5 5 4 4 6 7 8 7 6 6 19/30 Read block 573 len 20480 20/30 Write block 275 len 16384 data: 7 7 7 8 21/30 Write block 389 len 16384 data: 5 6 6 6 22/30 Write block 255 len 28672 data: 5 7 7 7 7 6 5 23/30 Write block 223 len 8192 data: 5 5 24/30 Read block 52 len 40960 25/30 Read block 567 len 20480 26/30 Flush 27/30 Write block 139 len 4096 data: 7 28/30 Write block 34 len 8192 data: 7 6 29/30 Write block 379 len 40960 data: 5 7 5 5 5 5 5 6 7 6 30/30 Read block 178 len 20480 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:24.184725248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1408} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 52 52 52 81 81 81 T T T 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 56 56 56 89 89 89 F F F 6 120-139 56 56 56 89 89 89 T T T 7 140-159 56 56 56 89 89 89 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 56 56 56 89 89 89 F F F 12 240-259 56 56 56 89 89 89 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 56 56 56 89 89 89 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 52 52 52 81 81 81 F F F 18 360-379 56 56 56 89 89 89 T T T 19 380-399 56 56 56 89 89 89 T T T 20 400-419 52 52 52 81 81 81 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 55 55 55 87 87 87 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 52 52 52 81 81 81 F F F 27 540-559 52 52 52 81 81 81 F F F 28 560-579 56 56 56 88 88 88 F F F 29 580-599 55 55 55 87 87 87 F F F Max gen: 56, Max flush: 89 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 57 57 57 90 90 90 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 56 56 56 89 89 89 F F F 6 120-139 57 57 57 90 90 90 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 51 51 51 80 80 80 F F F 11 220-239 56 56 56 89 89 89 F F F 12 240-259 56 56 56 89 89 89 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 56 56 56 89 89 89 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 52 52 52 81 81 81 F F F 18 360-379 57 57 57 90 90 90 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 52 52 52 81 81 81 F F F 21 420-439 51 51 51 80 80 80 F F F 22 440-459 55 55 55 87 87 87 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 52 52 52 81 81 81 F F F 27 540-559 52 52 52 81 81 81 F F F 28 560-579 56 56 56 88 88 88 F F F 29 580-599 55 55 55 87 87 87 F F F Max gen: 57, Max flush: 90 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.42832357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.472809194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.47365786Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.47370391Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.473720386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.473734963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} {"msg":"Crucible 33ed19d5-d477-4170-aaf7-c1331ea26e72 has session id: 3e24fe3c-eca6-4673-8cf3-07e8c44d11d9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474286034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Upstairs opts: Upstairs UUID: 33ed19d5-d477-4170-aaf7-c1331ea26e72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474333214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Crucible stats registered with UUID: 33ed19d5-d477-4170-aaf7-c1331ea26e72","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474356598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:28.474377204Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474802742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474828526Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} {"msg":"33ed19d5-d477-4170-aaf7-c1331ea26e72 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474890823Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474913398Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474936093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.47494959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.474963497Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475052758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475094409Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475120153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475146447Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475172102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37256)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475199826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41630)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.47522829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47612)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.475257813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.476094792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.476262686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.476718987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.476870374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.476994887Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.477356779Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.477457667Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.477709662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[0]R flush_numbers[0..12]: [88, 90, 80, 61, 89, 89, 90, 89, 85, 81, 80, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478205485Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[0]R generation[0..12]: [56, 57, 51, 36, 56, 56, 57, 56, 54, 52, 51, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478223981Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478238088Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[1]R flush_numbers[0..12]: [88, 90, 80, 61, 89, 89, 90, 89, 85, 81, 80, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478251445Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[1]R generation[0..12]: [56, 57, 51, 36, 56, 56, 57, 56, 54, 52, 51, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478264532Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478278399Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[2]R flush_numbers[0..12]: [88, 90, 80, 61, 89, 89, 90, 89, 85, 81, 80, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478291646Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[2]R generation[0..12]: [56, 57, 51, 36, 56, 56, 57, 56, 54, 52, 51, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478304713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.4783181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478341295Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478355032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.47836743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478379767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"":"downstairs","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478392584Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478404362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"0","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478416919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"1","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478429516Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"client":"2","":"downstairs","session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478442164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478453701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"33ed19d5-d477-4170-aaf7-c1331ea26e72 is now active with session: 3e24fe3c-eca6-4673-8cf3-07e8c44d11d9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478466028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478477516Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416,"session_id":"3e24fe3c-eca6-4673-8cf3-07e8c44d11d9"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:28.478489143Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 250 len 24576 data: 3 4 3 4 4 6 02/30 Read block 457 len 4096 03/30 Write block 219 len 4096 data: 4 04/30 Read block 455 len 20480 05/30 Flush 06/30 Write block 420 len 8192 data: 6 5 07/30 Flush 08/30 Write block 202 len 28672 data: 6 6 6 7 8 9 9 09/30 Read block 13 len 36864 10/30 Read block 49 len 4096 11/30 Read block 309 len 32768 12/30 Read block 2 len 12288 13/30 Read block 208 len 36864 14/30 Read block 353 len 40960 15/30 Flush 16/30 Flush 17/30 Write block 367 len 32768 data: 2 3 3 4 4 4 5 4 18/30 Read block 496 len 16384 19/30 Write block 539 len 32768 data: 12 13 13 12 11 11 8 7 20/30 Flush 21/30 Write block 345 len 12288 data: 7 7 5 22/30 Read block 522 len 28672 23/30 Read block 28 len 12288 24/30 Write block 454 len 8192 data: 9 7 25/30 Write block 107 len 28672 data: 3 4 4 5 5 5 4 26/30 Flush 27/30 Write block 267 len 12288 data: 5 6 5 28/30 Write block 403 len 28672 data: 6 5 4 4 4 5 6 29/30 Write block 34 len 4096 data: 8 30/30 Read block 496 len 8192 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:28.498917018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1416} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 57 57 57 90 90 90 T T T 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 58 58 58 96 96 96 F F F 6 120-139 57 57 57 90 90 90 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 58 58 58 93 93 93 F F F 11 220-239 56 56 56 89 89 89 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 56 56 56 89 89 89 T T T 14 280-299 56 56 56 89 89 89 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 52 52 52 81 81 81 T T T 21 420-439 58 58 58 92 92 92 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 56 56 56 88 88 88 F F F 29 580-599 55 55 55 87 87 87 F F F Max gen: 58, Max flush: 96 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 59 59 59 97 97 97 F F F 2 040-059 51 51 51 80 80 80 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 58 58 58 96 96 96 F F F 6 120-139 57 57 57 90 90 90 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 58 58 58 93 93 93 F F F 11 220-239 56 56 56 89 89 89 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 59 59 59 97 97 97 F F F 14 280-299 56 56 56 89 89 89 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 54 54 54 86 86 86 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 59 59 59 97 97 97 F F F 21 420-439 58 58 58 92 92 92 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 56 56 56 88 88 88 F F F 29 580-599 55 55 55 87 87 87 F F F Max gen: 59, Max flush: 97 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.731310828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.776231737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.77704738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.77709118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.777106207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.777120704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} {"msg":"Crucible f47c7baf-62da-48e1-acb8-e86926d3c90d has session id: e62eb20d-36df-4616-95fd-da7a43255a09","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.777673234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Upstairs opts: Upstairs UUID: f47c7baf-62da-48e1-acb8-e86926d3c90d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.777715535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Crucible stats registered with UUID: f47c7baf-62da-48e1-acb8-e86926d3c90d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.777731872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:32.777749078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778146502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778174176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} {"msg":"f47c7baf-62da-48e1-acb8-e86926d3c90d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778252499Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778270545Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.77829382Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778308697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778322474Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778335631Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778350938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778373463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778395708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778418113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"ds_connection connected from Ok(127.0.0.1:53861)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778461024Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"ds_connection connected from Ok(127.0.0.1:42963)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778728236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"ds_connection connected from Ok(127.0.0.1:40720)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.778766178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.779191116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.779235366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.779385574Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.77991107Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.779940253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.77995619Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780540173Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780638592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[0]R flush_numbers[0..12]: [88, 97, 80, 61, 89, 96, 90, 89, 85, 81, 93, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780688061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[0]R generation[0..12]: [56, 59, 51, 36, 56, 58, 57, 56, 54, 52, 58, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780705558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780719774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[1]R flush_numbers[0..12]: [88, 97, 80, 61, 89, 96, 90, 89, 85, 81, 93, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780733252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[1]R generation[0..12]: [56, 59, 51, 36, 56, 58, 57, 56, 54, 52, 58, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780746909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780760676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[2]R flush_numbers[0..12]: [88, 97, 80, 61, 89, 96, 90, 89, 85, 81, 93, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780779492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[2]R generation[0..12]: [56, 59, 51, 36, 56, 58, 57, 56, 54, 52, 58, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780793438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780806856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780819363Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780839179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780852006Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780864753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"":"downstairs","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.78087744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780889888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"0","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780903025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"1","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780915582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"client":"2","":"downstairs","session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780928569Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780940537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"f47c7baf-62da-48e1-acb8-e86926d3c90d is now active with session: e62eb20d-36df-4616-95fd-da7a43255a09","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780953034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780964632Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424,"session_id":"e62eb20d-36df-4616-95fd-da7a43255a09"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:32.780978689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 139 len 16384 data: 8 8 5 6 02/30 Read block 380 len 28672 03/30 Write block 590 len 32768 data: 3 4 5 5 3 3 3 3 04/30 Read block 68 len 4096 05/30 Read block 55 len 4096 06/30 Flush 07/30 Write block 54 len 20480 data: 6 5 6 7 8 08/30 Read block 76 len 4096 09/30 Read block 113 len 28672 10/30 Read block 121 len 12288 11/30 Read block 518 len 16384 12/30 Write block 128 len 40960 data: 3 3 3 4 5 7 9 9 8 5 13/30 Write block 21 len 28672 data: 5 6 7 7 6 7 9 14/30 Read block 549 len 20480 15/30 Write block 563 len 20480 data: 2 2 4 5 6 16/30 Write block 279 len 24576 data: 7 7 7 6 6 6 17/30 Read block 262 len 28672 18/30 Write block 167 len 36864 data: 4 4 4 3 5 6 6 7 9 19/30 Flush 20/30 Flush 21/30 Write block 121 len 8192 data: 9 8 22/30 Write block 330 len 36864 data: 6 6 6 4 4 3 3 3 4 23/30 Write block 150 len 24576 data: 8 6 5 5 4 3 24/30 Read block 413 len 36864 25/30 Flush 26/30 Write block 407 len 24576 data: 5 6 7 6 6 6 27/30 Read block 204 len 20480 28/30 Read block 62 len 8192 29/30 Write block 210 len 16384 data: 5 6 5 5 30/30 Read block 157 len 16384 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:32.802192643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1424} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 60 60 60 99 99 99 F F F 2 040-059 60 60 60 99 99 99 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 58 58 58 96 96 96 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 60 60 60 99 99 99 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 58 58 58 93 93 93 T T T 11 220-239 56 56 56 89 89 89 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 60 60 60 99 99 99 F F F 14 280-299 60 60 60 99 99 99 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 59 59 59 97 97 97 T T T 21 420-439 58 58 58 92 92 92 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 60 60 60 99 99 99 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 60, Max flush: 101 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 88 88 88 F F F 1 020-039 60 60 60 99 99 99 F F F 2 040-059 60 60 60 99 99 99 F F F 3 060-079 36 36 36 61 61 61 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 58 58 58 96 96 96 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 60 60 60 99 99 99 F F F 9 180-199 52 52 52 81 81 81 F F F 10 200-219 61 61 61 102 102 102 F F F 11 220-239 56 56 56 89 89 89 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 60 60 60 99 99 99 F F F 14 280-299 60 60 60 99 99 99 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 61 61 61 102 102 102 F F F 21 420-439 58 58 58 92 92 92 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 60 60 60 99 99 99 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 61, Max flush: 102 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.03224369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.076158867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.076963673Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.077036627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.077056333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.07707219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} {"msg":"Crucible 8ad7470e-33b3-4cd2-af25-5ec591dd84f7 has session id: cecf080f-0d5f-4355-a593-4035085d1ef5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.077651524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Upstairs opts: Upstairs UUID: 8ad7470e-33b3-4cd2-af25-5ec591dd84f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.077698924Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Crucible stats registered with UUID: 8ad7470e-33b3-4cd2-af25-5ec591dd84f7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.07771556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:37.077728317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078150886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.0781763Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} {"msg":"8ad7470e-33b3-4cd2-af25-5ec591dd84f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078244165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078261712Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078285077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078298924Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078313011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078326178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078340165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078352922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078368649Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078399022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35211)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078503839Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45382)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078540431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47992)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.078568985Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.079220594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.079261555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.079924202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.079951955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.080281774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.080599895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.08062577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.080861418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[0]R flush_numbers[0..12]: [88, 99, 99, 61, 89, 96, 101, 101, 99, 81, 102, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081355482Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[0]R generation[0..12]: [56, 60, 60, 36, 56, 58, 60, 60, 60, 52, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081376837Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081391634Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[1]R flush_numbers[0..12]: [88, 99, 99, 61, 89, 96, 101, 101, 99, 81, 102, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.08140838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[1]R generation[0..12]: [56, 60, 60, 36, 56, 58, 60, 60, 60, 52, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081422117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081435714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[2]R flush_numbers[0..12]: [88, 99, 99, 61, 89, 96, 101, 101, 99, 81, 102, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081448881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[2]R generation[0..12]: [56, 60, 60, 36, 56, 58, 60, 60, 60, 52, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081462128Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081475965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081488453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081507009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081519456Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081531963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"":"downstairs","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081544581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081556078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"0","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081569105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"1","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081582102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"client":"2","":"downstairs","session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081598739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081611186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"8ad7470e-33b3-4cd2-af25-5ec591dd84f7 is now active with session: cecf080f-0d5f-4355-a593-4035085d1ef5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081623793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:37.081635381Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432,"session_id":"cecf080f-0d5f-4355-a593-4035085d1ef5"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:40:37.081647098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 20 len 8192 data: 6 6 02/30 Read block 187 len 36864 03/30 Read block 264 len 12288 04/30 Read block 88 len 12288 05/30 Write block 59 len 32768 data: 7 9 10 10 7 5 5 7 06/30 Write block 223 len 4096 data: 6 07/30 Read block 487 len 24576 08/30 Read block 69 len 16384 09/30 Read block 333 len 32768 10/30 Read block 43 len 28672 11/30 Read block 182 len 4096 12/30 Flush 13/30 Flush 14/30 Write block 437 len 8192 data: 4 5 15/30 Write block 30 len 4096 data: 6 16/30 Flush 17/30 Write block 0 len 16384 data: 2 2 2 2 18/30 Flush 19/30 Flush 20/30 Write block 563 len 8192 data: 3 3 21/30 Read block 153 len 32768 22/30 Read block 526 len 36864 23/30 Read block 414 len 4096 24/30 Write block 170 len 20480 data: 4 6 7 7 8 25/30 Flush 26/30 Flush 27/30 Write block 105 len 40960 data: 5 5 4 5 5 6 6 6 5 4 28/30 Write block 267 len 24576 data: 6 7 6 4 5 6 29/30 Write block 195 len 36864 data: 5 2 2 2 4 4 4 7 7 30/30 Read block 505 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 4 2 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:37.100422621Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1432} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 62 62 62 103 103 103 F F F 3 060-079 62 62 62 103 103 103 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 58 58 58 96 96 96 T T T 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 52 52 52 81 81 81 T T T 10 200-219 61 61 61 102 102 102 T T T 11 220-239 62 62 62 103 103 103 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 60 60 60 99 99 99 T T T 14 280-299 60 60 60 99 99 99 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 61 61 61 102 102 102 F F F 21 420-439 62 62 62 105 105 105 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 62 62 62 108 108 108 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 62, Max flush: 108 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 62 62 62 103 103 103 F F F 3 060-079 62 62 62 103 103 103 F F F 4 080-099 56 56 56 89 89 89 F F F 5 100-119 63 63 63 109 109 109 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 60 60 60 99 99 99 F F F 15 300-319 54 54 54 86 86 86 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 58 58 58 96 96 96 F F F 18 360-379 58 58 58 95 95 95 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 61 61 61 102 102 102 F F F 21 420-439 62 62 62 105 105 105 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 54 54 54 86 86 86 F F F 25 500-519 51 51 51 80 80 80 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 62 62 62 108 108 108 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.337040615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.380960881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.381829333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.381875933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.381896358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.381912745Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} {"msg":"Crucible 9b8dea65-a308-409f-bbfe-705450ba04da has session id: 133bdf31-babf-4f22-a539-452d9b6cc969","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.382497448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Upstairs opts: Upstairs UUID: 9b8dea65-a308-409f-bbfe-705450ba04da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.382542328Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Crucible stats registered with UUID: 9b8dea65-a308-409f-bbfe-705450ba04da","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.382565573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:41.382586699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.382989282Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383034262Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} {"msg":"9b8dea65-a308-409f-bbfe-705450ba04da active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.38308876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383117334Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383160194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383191988Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383221911Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383299654Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383335017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383359151Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383381977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383406041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"ds_connection connected from Ok(127.0.0.1:63869)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383510479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"ds_connection connected from Ok(127.0.0.1:41271)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383559418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"ds_connection connected from Ok(127.0.0.1:54564)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.383588782Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.384079135Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.384112038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.384529478Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.384815556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.384855917Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385153613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385342002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385411177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[0]R flush_numbers[0..12]: [106, 105, 103, 103, 89, 109, 101, 101, 108, 109, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385704723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[0]R generation[0..12]: [62, 62, 62, 62, 56, 63, 60, 60, 62, 63, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385729788Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385745425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[1]R flush_numbers[0..12]: [106, 105, 103, 103, 89, 109, 101, 101, 108, 109, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385759821Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[1]R generation[0..12]: [62, 62, 62, 62, 56, 63, 60, 60, 62, 63, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385773408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385787025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[2]R flush_numbers[0..12]: [106, 105, 103, 103, 89, 109, 101, 101, 108, 109, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385800222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[2]R generation[0..12]: [62, 62, 62, 62, 56, 63, 60, 60, 62, 63, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.3858132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385827557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385845803Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.3858589Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385871437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385883935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"":"downstairs","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385900611Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385912678Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"0","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385928965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"1","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385946431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"client":"2","":"downstairs","session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385959658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.385971416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"9b8dea65-a308-409f-bbfe-705450ba04da is now active with session: 133bdf31-babf-4f22-a539-452d9b6cc969","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2025-01-23T17:40:41.385987362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.386006258Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440,"session_id":"133bdf31-babf-4f22-a539-452d9b6cc969"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:41.386023344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 32768 data: 6 5 6 4 5 5 5 5 02/30 Write block 443 len 40960 data: 4 5 7 7 5 7 6 7 7 7 03/30 Write block 411 len 8192 data: 7 7 04/30 Read block 10 len 32768 05/30 Write block 297 len 36864 data: 5 4 2 3 5 6 7 7 6 06/30 Write block 502 len 40960 data: 7 7 6 7 7 5 6 6 6 4 07/30 Write block 82 len 40960 data: 8 8 8 8 6 6 5 5 4 4 08/30 Read block 67 len 36864 09/30 Read block 521 len 32768 10/30 Read block 557 len 4096 11/30 Read block 179 len 12288 12/30 Write block 195 len 16384 data: 6 3 3 3 13/30 Write block 349 len 32768 data: 5 5 4 4 4 4 4 5 14/30 Read block 372 len 4096 15/30 Read block 49 len 24576 16/30 Write block 427 len 4096 data: 6 17/30 Read block 332 len 36864 18/30 Write block 456 len 12288 data: 7 6 6 19/30 Read block 363 len 28672 20/30 Write block 363 len 8192 data: 2 2 21/30 Write block 575 len 32768 data: 5 5 5 6 7 6 6 6 22/30 Write block 485 len 20480 data: 6 4 3 4 4 23/30 Flush 24/30 Write block 563 len 20480 data: 4 4 5 6 7 25/30 Read block 306 len 20480 26/30 Read block 178 len 16384 27/30 Read block 31 len 4096 28/30 Write block 359 len 36864 data: 2 2 2 2 3 3 2 2 3 29/30 Read block 217 len 8192 30/30 Read block 212 len 4096 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:41.405582397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1440} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 62 62 62 103 103 103 F F F 3 060-079 62 62 62 103 103 103 F F F 4 080-099 64 64 64 110 110 110 F F F 5 100-119 63 63 63 109 109 109 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 64 64 64 110 110 110 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 64 64 64 110 110 110 T T T 18 360-379 64 64 64 110 110 110 T T T 19 380-399 57 57 57 90 90 90 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 64 64 64 110 110 110 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 64 64 64 110 110 110 F F F 28 560-579 64 64 64 110 110 110 T T T 29 580-599 64 64 64 110 110 110 F F F Max gen: 64, Max flush: 110 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 62 62 62 103 103 103 F F F 3 060-079 62 62 62 103 103 103 F F F 4 080-099 64 64 64 110 110 110 F F F 5 100-119 63 63 63 109 109 109 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 64 64 64 110 110 110 F F F 16 320-339 60 60 60 101 101 101 F F F 17 340-359 65 65 65 111 111 111 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 57 57 57 90 90 90 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 56 56 56 88 88 88 F F F 24 480-499 64 64 64 110 110 110 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 64 64 64 110 110 110 F F F 28 560-579 65 65 65 111 111 111 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 65, Max flush: 111 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.63196132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.677374272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678192715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678238175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678254762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678269888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} {"msg":"Crucible 360e82d8-8ac1-44ab-8d58-a0985c356cde has session id: 78e37437-ff94-4eb6-8b29-58c8f50cfde3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678757233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Upstairs opts: Upstairs UUID: 360e82d8-8ac1-44ab-8d58-a0985c356cde, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.678798284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Crucible stats registered with UUID: 360e82d8-8ac1-44ab-8d58-a0985c356cde","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.67881473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:45.678827678Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679328449Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679353973Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} {"msg":"360e82d8-8ac1-44ab-8d58-a0985c356cde active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679428028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679446833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679470578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679484745Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679499322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679513049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679527406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679549001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679572316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679596081Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58175)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679644701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40723)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679682162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34915)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.679710136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68047845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.680507104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68121654Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68126162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.681900622Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.682032573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.682445284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.682946095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[0]R flush_numbers[0..12]: [106, 105, 103, 103, 110, 109, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683437709Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[0]R generation[0..12]: [62, 62, 62, 62, 64, 63, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683456385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683471152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[1]R flush_numbers[0..12]: [106, 105, 103, 103, 110, 109, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683485298Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[1]R generation[0..12]: [62, 62, 62, 62, 64, 63, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683498966Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683512513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[2]R flush_numbers[0..12]: [106, 105, 103, 103, 110, 109, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68352712Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[2]R generation[0..12]: [62, 62, 62, 62, 64, 63, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683540317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683558203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68357108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:40:45.683583897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683599674Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683611901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"":"downstairs","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683624339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683636616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"0","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683649753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"1","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68366221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"client":"2","":"downstairs","session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683674848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683686465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"360e82d8-8ac1-44ab-8d58-a0985c356cde is now active with session: 78e37437-ff94-4eb6-8b29-58c8f50cfde3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683698702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.68370991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448,"session_id":"78e37437-ff94-4eb6-8b29-58c8f50cfde3"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:45.683721657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 573 len 4096 data: 6 02/30 Read block 390 len 4096 03/30 Write block 380 len 40960 data: 8 6 6 6 6 6 7 8 7 6 04/30 Write block 278 len 24576 data: 9 8 8 8 7 7 05/30 Write block 519 len 32768 data: 3 4 5 5 5 5 7 7 06/30 Read block 340 len 12288 07/30 Write block 475 len 20480 data: 3 3 3 3 4 08/30 Write block 0 len 20480 data: 3 3 3 3 4 09/30 Write block 107 len 8192 data: 5 6 10/30 Write block 471 len 32768 data: 8 8 5 4 4 4 4 4 11/30 Read block 352 len 16384 12/30 Write block 257 len 32768 data: 8 8 8 7 6 5 5 6 13/30 Read block 373 len 16384 14/30 Read block 304 len 4096 15/30 Flush 16/30 Write block 85 len 20480 data: 9 7 7 6 6 17/30 Write block 382 len 16384 data: 7 7 7 7 18/30 Read block 547 len 4096 19/30 Write block 312 len 40960 data: 6 7 5 5 6 5 5 5 5 5 20/30 Read block 485 len 28672 21/30 Read block 285 len 24576 22/30 Write block 383 len 8192 data: 8 8 23/30 Write block 69 len 20480 data: 5 4 4 5 5 24/30 Read block 135 len 40960 25/30 Read block 79 len 28672 26/30 Write block 44 len 8192 data: 3 5 27/30 Read block 415 len 36864 28/30 Read block 366 len 16384 29/30 Read block 388 len 8192 30/30 Write block 497 len 36864 data: 6 6 5 7 6 8 8 7 8 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:45.703516719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1448} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 62 62 62 103 103 103 T T T 3 060-079 62 62 62 103 103 103 T T T 4 080-099 64 64 64 110 110 110 T T T 5 100-119 66 66 66 112 112 112 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 66 66 66 112 112 112 F F F 14 280-299 66 66 66 112 112 112 F F F 15 300-319 64 64 64 110 110 110 T T T 16 320-339 60 60 60 101 101 101 T T T 17 340-359 65 65 65 111 111 111 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 66 66 66 112 112 112 T T T 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 64 64 64 110 110 110 T T T 25 500-519 66 66 66 112 112 112 T T T 26 520-539 66 66 66 112 112 112 F F F 27 540-559 64 64 64 110 110 110 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 66, Max flush: 112 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 62 62 62 105 105 105 F F F 2 040-059 67 67 67 113 113 113 F F F 3 060-079 67 67 67 113 113 113 F F F 4 080-099 67 67 67 113 113 113 F F F 5 100-119 66 66 66 112 112 112 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 66 66 66 112 112 112 F F F 14 280-299 66 66 66 112 112 112 F F F 15 300-319 67 67 67 113 113 113 F F F 16 320-339 67 67 67 113 113 113 F F F 17 340-359 65 65 65 111 111 111 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 67 67 67 113 113 113 F F F 25 500-519 67 67 67 113 113 113 F F F 26 520-539 66 66 66 112 112 112 F F F 27 540-559 64 64 64 110 110 110 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 67, Max flush: 113 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:49.960689121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.003609753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004254384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004298304Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004313501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004328068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} {"msg":"Crucible 864f0f29-2120-41ba-9470-60c419d77123 has session id: 22751d89-eb17-4da7-8fa8-b3b829678256","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004830369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Upstairs opts: Upstairs UUID: 864f0f29-2120-41ba-9470-60c419d77123, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004878988Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Crucible stats registered with UUID: 864f0f29-2120-41ba-9470-60c419d77123","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.004905472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:50.004927278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005400165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005432758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} {"msg":"864f0f29-2120-41ba-9470-60c419d77123 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005494805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005521139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005545714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.00556123Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005576287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005589174Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005603001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005616038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005642003Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005670427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"ds_connection connected from Ok(127.0.0.1:52740)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005708909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"ds_connection connected from Ok(127.0.0.1:55622)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005755488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"ds_connection connected from Ok(127.0.0.1:64011)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.005783112Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.006407687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.006436091Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.007106276Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.007147277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.007700067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.007815642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.007854434Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008358555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[0]R flush_numbers[0..12]: [112, 105, 113, 113, 113, 112, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008821694Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[0]R generation[0..12]: [66, 62, 67, 67, 67, 66, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008846589Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008862635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[1]R flush_numbers[0..12]: [112, 105, 113, 113, 113, 112, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008877292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[1]R generation[0..12]: [66, 62, 67, 67, 67, 66, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008891309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008905916Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[2]R flush_numbers[0..12]: [112, 105, 113, 113, 113, 112, 101, 101, 108, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008920373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[2]R generation[0..12]: [66, 62, 67, 67, 67, 66, 60, 60, 62, 64, 63, 62]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:40:50.008935239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008953886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008967233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.00898019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.008993487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009006214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"":"downstairs","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009019681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009031329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"0","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009045136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"1","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009060762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"client":"2","":"downstairs","session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009074819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009086997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"864f0f29-2120-41ba-9470-60c419d77123 is now active with session: 22751d89-eb17-4da7-8fa8-b3b829678256","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009100144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:50.009112821Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456,"session_id":"22751d89-eb17-4da7-8fa8-b3b829678256"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2025-01-23T17:40:50.009124878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 565 len 36864 data: 6 7 8 6 7 5 5 6 7 03/30 Write block 550 len 40960 data: 7 5 6 6 6 6 6 6 6 6 04/30 Read block 149 len 20480 05/30 Read block 404 len 20480 06/30 Read block 163 len 28672 07/30 Write block 295 len 8192 data: 6 6 08/30 Flush 09/30 Write block 353 len 4096 data: 5 10/30 Read block 297 len 12288 11/30 Read block 437 len 24576 12/30 Flush 13/30 Write block 105 len 32768 data: 6 6 6 7 6 7 7 7 14/30 Write block 556 len 28672 data: 7 7 7 7 6 5 4 15/30 Write block 306 len 12288 data: 5 4 5 16/30 Read block 276 len 32768 17/30 Write block 321 len 20480 data: 6 4 5 6 6 18/30 Write block 69 len 28672 data: 6 5 5 6 6 4 4 19/30 Flush 20/30 Write block 39 len 8192 data: 8 7 21/30 Write block 55 len 4096 data: 6 22/30 Write block 529 len 36864 data: 9 8 9 8 9 8 6 4 8 23/30 Write block 339 len 8192 data: 5 8 24/30 Read block 112 len 28672 25/30 Flush 26/30 Write block 159 len 32768 data: 10 7 7 7 6 6 4 5 27/30 Read block 206 len 12288 28/30 Write block 262 len 28672 data: 6 6 7 6 5 7 8 29/30 Read block 215 len 8192 30/30 Read block 58 len 28672 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:50.027917958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1456} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 67 67 67 113 113 113 F F F 5 100-119 68 68 68 117 117 117 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 60 60 60 101 101 101 T T T 8 160-179 62 62 62 108 108 108 T T T 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 66 66 66 112 112 112 T T T 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 68 68 68 118 118 118 F F F 17 340-359 68 68 68 118 118 118 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 67 67 67 113 113 113 F F F 25 500-519 67 67 67 113 113 113 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 68 68 68 117 117 117 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 68, Max flush: 118 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 67 67 67 113 113 113 F F F 5 100-119 68 68 68 117 117 117 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 69 69 69 119 119 119 F F F 8 160-179 69 69 69 119 119 119 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 69 69 69 119 119 119 F F F 14 280-299 68 68 68 115 115 115 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 68 68 68 118 118 118 F F F 17 340-359 68 68 68 118 118 118 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 67 67 67 113 113 113 F F F 25 500-519 67 67 67 113 113 113 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 68 68 68 117 117 117 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 69, Max flush: 119 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.253567458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.297759085Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.298743242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.298790412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.298806259Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.298821145Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} {"msg":"Crucible 4a95db33-20e0-4bb0-8e93-e6d154aba661 has session id: c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299374465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Upstairs opts: Upstairs UUID: 4a95db33-20e0-4bb0-8e93-e6d154aba661, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299423435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Crucible stats registered with UUID: 4a95db33-20e0-4bb0-8e93-e6d154aba661","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299450489Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:54.299472764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299894913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299923426Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} {"msg":"4a95db33-20e0-4bb0-8e93-e6d154aba661 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299936494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299949041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.299984173Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300016496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.30004334Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300067615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.30009318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300109876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300134971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300161035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:60956)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300242957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:41437)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300282968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:57522)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300308543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.300965851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.301072157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.301233523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.301731195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.301764778Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.301798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302391742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302426054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[0]R flush_numbers[0..12]: [112, 118, 118, 117, 113, 117, 101, 119, 119, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302453518Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[0]R generation[0..12]: [66, 68, 68, 68, 67, 68, 60, 69, 69, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302470225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302485641Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[1]R flush_numbers[0..12]: [112, 118, 118, 117, 113, 117, 101, 119, 119, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302499558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[1]R generation[0..12]: [66, 68, 68, 68, 67, 68, 60, 69, 69, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302512805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302526433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[2]R flush_numbers[0..12]: [112, 118, 118, 117, 113, 117, 101, 119, 119, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302540839Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[2]R generation[0..12]: [66, 68, 68, 68, 67, 68, 60, 69, 69, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302554007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302567664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302580281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:40:54.302605036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.30263183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302655805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"":"downstairs","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302672381Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302684488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"0","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302697805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"1","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302710723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"client":"2","":"downstairs","session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.30272359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302736407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"4a95db33-20e0-4bb0-8e93-e6d154aba661 is now active with session: c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302748974Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302760482Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464,"session_id":"c4aaab12-7545-4e0e-b1f0-6f9bcd21e8ad"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:54.302772299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 281 len 12288 data: 9 8 8 02/30 Write block 310 len 32768 data: 7 7 7 8 6 6 7 6 03/30 Read block 518 len 20480 04/30 Read block 231 len 4096 05/30 Read block 341 len 12288 06/30 Read block 358 len 8192 07/30 Read block 65 len 4096 08/30 Read block 139 len 40960 09/30 Read block 466 len 12288 10/30 Write block 265 len 8192 data: 7 6 11/30 Write block 31 len 20480 data: 5 6 5 9 7 12/30 Read block 79 len 40960 13/30 Read block 242 len 40960 14/30 Write block 85 len 28672 data: 10 8 8 7 7 5 5 15/30 Write block 569 len 4096 data: 8 16/30 Write block 339 len 32768 data: 6 9 6 9 8 7 8 8 17/30 Write block 168 len 16384 data: 5 5 5 7 18/30 Read block 383 len 36864 19/30 Read block 301 len 24576 20/30 Read block 515 len 16384 21/30 Read block 15 len 28672 22/30 Read block 111 len 20480 23/30 Write block 16 len 4096 data: 7 24/30 Write block 445 len 24576 data: 8 8 6 8 7 8 25/30 Read block 267 len 20480 26/30 Read block 231 len 20480 27/30 Write block 284 len 8192 data: 7 6 28/30 Write block 511 len 12288 data: 5 4 3 29/30 Read block 134 len 12288 30/30 Write block 286 len 12288 data: 5 5 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:54.321577946Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1464} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 T T T 1 020-039 68 68 68 118 118 118 T T T 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 67 67 67 113 113 113 T T T 5 100-119 68 68 68 117 117 117 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 69 69 69 119 119 119 F F F 8 160-179 69 69 69 119 119 119 T T T 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 69 69 69 119 119 119 T T T 14 280-299 68 68 68 115 115 115 T T T 15 300-319 68 68 68 117 117 117 T T T 16 320-339 68 68 68 118 118 118 T T T 17 340-359 68 68 68 118 118 118 T T T 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 64 64 64 110 110 110 T T T 23 460-479 66 66 66 112 112 112 F F F 24 480-499 67 67 67 113 113 113 F F F 25 500-519 67 67 67 113 113 113 T T T 26 520-539 68 68 68 118 118 118 F F F 27 540-559 68 68 68 117 117 117 F F F 28 560-579 68 68 68 117 117 117 T T T 29 580-599 64 64 64 110 110 110 F F F Max gen: 69, Max flush: 119 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 120 120 120 F F F 1 020-039 70 70 70 120 120 120 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 70 70 70 120 120 120 F F F 5 100-119 68 68 68 117 117 117 F F F 6 120-139 60 60 60 101 101 101 F F F 7 140-159 69 69 69 119 119 119 F F F 8 160-179 70 70 70 120 120 120 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 63 63 63 109 109 109 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 70 70 70 120 120 120 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 70 70 70 120 120 120 F F F 17 340-359 70 70 70 120 120 120 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 70 70 70 120 120 120 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 67 67 67 113 113 113 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 68 68 68 117 117 117 F F F 28 560-579 70 70 70 120 120 120 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 70, Max flush: 120 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.604007687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:58.648045697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.648502368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.68002083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.680711691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.680746693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.680764999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.680780086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} {"msg":"Crucible c1451161-830d-4572-9590-0a690036d02a has session id: fb1baada-c063-4bdb-9b1d-d1d668643327","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.681448321Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Upstairs opts: Upstairs UUID: c1451161-830d-4572-9590-0a690036d02a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.681492011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Crucible stats registered with UUID: c1451161-830d-4572-9590-0a690036d02a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.681519165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:58.681540111Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.681921928Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.681944703Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} {"msg":"c1451161-830d-4572-9590-0a690036d02a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682022707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682044432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682080374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682109878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682136282Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682150579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682170824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682203017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682232331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682260835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"ds_connection connected from Ok(127.0.0.1:50011)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682287209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"ds_connection connected from Ok(127.0.0.1:61064)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682314843Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"ds_connection connected from Ok(127.0.0.1:37535)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.682343087Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.683057842Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.683220207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.683325624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.683725718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.683822037Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684052367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684343024Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684435334Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[0]R flush_numbers[0..12]: [120, 120, 118, 117, 120, 117, 101, 119, 120, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684587591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[0]R generation[0..12]: [70, 70, 68, 68, 70, 68, 60, 69, 70, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684617534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684633981Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[1]R flush_numbers[0..12]: [120, 120, 118, 117, 120, 117, 101, 119, 120, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684648198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[1]R generation[0..12]: [70, 70, 68, 68, 70, 68, 60, 69, 70, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684661445Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684675002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[2]R flush_numbers[0..12]: [120, 120, 118, 117, 120, 117, 101, 119, 120, 110, 109, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684688349Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"[2]R generation[0..12]: [70, 70, 68, 68, 70, 68, 60, 69, 70, 64, 63, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684701726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684721272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Max found gen is 71","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684734169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Generation requested: 72 >= found:71","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684746846Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684759783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684772771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"":"downstairs","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684785648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684797206Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"0","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684812492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"1","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684825879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"client":"2","":"downstairs","session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684839106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684851084Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"c1451161-830d-4572-9590-0a690036d02a is now active with session: fb1baada-c063-4bdb-9b1d-d1d668643327","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684863621Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684875509Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472,"session_id":"fb1baada-c063-4bdb-9b1d-d1d668643327"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:40:58.684887486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 211 len 8192 data: 7 6 02/30 Write block 242 len 16384 data: 7 6 4 3 03/30 Flush 04/30 Write block 209 len 16384 data: 5 6 8 7 05/30 Read block 340 len 8192 06/30 Read block 164 len 24576 07/30 Read block 49 len 8192 08/30 Read block 303 len 12288 09/30 Write block 84 len 20480 data: 9 11 9 9 8 10/30 Read block 428 len 4096 11/30 Read block 40 len 24576 12/30 Write block 118 len 32768 data: 5 7 8 10 9 7 7 5 13/30 Read block 180 len 20480 14/30 Write block 157 len 36864 data: 7 7 11 8 8 8 7 7 5 15/30 Read block 509 len 12288 16/30 Flush 17/30 Write block 546 len 12288 data: 8 5 7 18/30 Read block 339 len 8192 19/30 Write block 1 len 24576 data: 4 4 4 5 5 5 20/30 Read block 302 len 24576 21/30 Read block 219 len 24576 22/30 Read block 223 len 8192 23/30 Write block 127 len 12288 data: 3 4 4 24/30 Flush 25/30 Write block 474 len 28672 data: 5 5 5 5 5 5 7 26/30 Read block 104 len 36864 27/30 Read block 276 len 36864 28/30 Read block 147 len 12288 29/30 Write block 6 len 32768 data: 6 5 7 6 8 8 9 9 30/30 Write block 282 len 24576 data: 9 9 8 7 6 6 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:40:58.705483534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1472} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 123 123 123 T T T 1 020-039 70 70 70 120 120 120 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 72 72 72 122 122 122 F F F 5 100-119 72 72 72 122 122 122 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 72 72 72 122 122 122 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 72 72 72 121 121 121 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 70 70 70 120 120 120 T T T 15 300-319 70 70 70 120 120 120 F F F 16 320-339 70 70 70 120 120 120 F F F 17 340-359 70 70 70 120 120 120 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 70 70 70 120 120 120 F F F 23 460-479 66 66 66 112 112 112 T T T 24 480-499 67 67 67 113 113 113 T T T 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 70 70 70 120 120 120 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 72, Max flush: 123 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 124 124 124 F F F 1 020-039 70 70 70 120 120 120 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 72 72 72 122 122 122 F F F 5 100-119 72 72 72 122 122 122 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 72 72 72 122 122 122 F F F 11 220-239 62 62 62 103 103 103 F F F 12 240-259 72 72 72 121 121 121 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 73 73 73 124 124 124 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 70 70 70 120 120 120 F F F 17 340-359 70 70 70 120 120 120 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 70 70 70 120 120 120 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 70 70 70 120 120 120 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 73, Max flush: 124 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:02.976047315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.019804616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.020535267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.020588336Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.020603393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02061798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} {"msg":"Crucible 3c52af95-56b5-4a4f-8400-f2025df56d1e has session id: 35405c21-fbf8-48bd-a594-53a0d2b286b8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02121648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Upstairs opts: Upstairs UUID: 3c52af95-56b5-4a4f-8400-f2025df56d1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021265339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Crucible stats registered with UUID: 3c52af95-56b5-4a4f-8400-f2025df56d1e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021284735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:03.021299172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02172264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021747595Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} {"msg":"3c52af95-56b5-4a4f-8400-f2025df56d1e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021808851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021835266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021867929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021901371Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.021929855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022020836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022067686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02209155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022114705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022180511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59875)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022222792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38593)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022253315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50878)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022280799Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022829741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.022858004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.023486728Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.023609581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.024118971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.024145635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.024160942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.024655005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[0]R flush_numbers[0..12]: [124, 120, 118, 117, 122, 122, 123, 122, 122, 110, 122, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025128572Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[0]R generation[0..12]: [73, 70, 68, 68, 72, 72, 72, 72, 72, 64, 72, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025148478Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025163355Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[1]R flush_numbers[0..12]: [124, 120, 118, 117, 122, 122, 123, 122, 122, 110, 122, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025177432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[1]R generation[0..12]: [73, 70, 68, 68, 72, 72, 72, 72, 72, 64, 72, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025191049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025204526Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[2]R flush_numbers[0..12]: [124, 120, 118, 117, 122, 122, 123, 122, 122, 110, 122, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025218283Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[2]R generation[0..12]: [73, 70, 68, 68, 72, 72, 72, 72, 72, 64, 72, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02523168Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025245587Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025258244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Generation requested: 74 >= found:74","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-23T17:41:03.025281389Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025295176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025308124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"":"downstairs","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025321311Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025333228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"0","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025346205Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"1","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025358942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"client":"2","":"downstairs","session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.02537206Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025384127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"3c52af95-56b5-4a4f-8400-f2025df56d1e is now active with session: 35405c21-fbf8-48bd-a594-53a0d2b286b8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025397004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025409182Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480,"session_id":"35405c21-fbf8-48bd-a594-53a0d2b286b8"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:03.025421549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 32768 data: 9 6 5 6 6 5 5 6 02/30 Write block 14 len 12288 data: 7 7 8 03/30 Flush 04/30 Write block 337 len 20480 data: 4 5 7 10 7 05/30 Write block 564 len 36864 data: 5 7 8 9 7 9 6 6 7 06/30 Read block 335 len 12288 07/30 Read block 427 len 36864 08/30 Read block 43 len 28672 09/30 Read block 260 len 36864 10/30 Read block 129 len 24576 11/30 Write block 85 len 4096 data: 12 12/30 Read block 422 len 12288 13/30 Read block 397 len 4096 14/30 Write block 459 len 24576 data: 6 5 5 5 4 4 15/30 Read block 244 len 32768 16/30 Read block 542 len 24576 17/30 Write block 257 len 8192 data: 9 9 18/30 Write block 291 len 28672 data: 7 6 5 7 7 7 6 19/30 Read block 345 len 8192 20/30 Read block 534 len 24576 21/30 Write block 469 len 20480 data: 7 7 9 9 6 22/30 Read block 27 len 4096 23/30 Flush 24/30 Write block 338 len 4096 data: 6 25/30 Flush 26/30 Write block 365 len 16384 data: 3 3 4 4 27/30 Write block 504 len 16384 data: 8 9 8 6 28/30 Write block 225 len 16384 data: 3 5 4 5 29/30 Write block 11 len 32768 data: 9 10 10 8 8 9 7 6 30/30 Read block 317 len 8192 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:03.044332972Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1480} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 125 125 125 T T T 1 020-039 70 70 70 120 120 120 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 74 74 74 126 126 126 F F F 5 100-119 72 72 72 122 122 122 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 72 72 72 122 122 122 F F F 11 220-239 62 62 62 103 103 103 T T T 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 74 74 74 126 126 126 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 65 65 65 111 111 111 T T T 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 74 74 74 126 126 126 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 70 70 70 120 120 120 T T T 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 74 74 74 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 74, Max flush: 127 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 128 128 128 F F F 1 020-039 70 70 70 120 120 120 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 74 74 74 126 126 126 F F F 5 100-119 72 72 72 122 122 122 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 72 72 72 122 122 122 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 74 74 74 126 126 126 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 64 64 64 110 110 110 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 74 74 74 126 126 126 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 74 74 74 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.283909606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.32784155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.328564903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.328608643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.32862397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.328639147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} {"msg":"Crucible ad1a1994-52db-4ca7-b9a8-cf67fe2cdfb6 has session id: 8e85bf7b-df28-4594-8734-a149efd5e17a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329273849Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Upstairs opts: Upstairs UUID: ad1a1994-52db-4ca7-b9a8-cf67fe2cdfb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329310301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Crucible stats registered with UUID: ad1a1994-52db-4ca7-b9a8-cf67fe2cdfb6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329325268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:07.329343554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329860732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329884647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} {"msg":"ad1a1994-52db-4ca7-b9a8-cf67fe2cdfb6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329954582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329972298Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.329995623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330019778Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330046182Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330073576Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330129424Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330160687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330185452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330211096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58478)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330235351Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40579)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330259416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34595)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33028407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.330845719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33088485Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33139353Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.331548487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.332026393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.332141718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.332169802Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.332634761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[0]R flush_numbers[0..12]: [128, 120, 118, 117, 126, 122, 123, 122, 122, 110, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333117137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[0]R generation[0..12]: [75, 70, 68, 68, 74, 72, 72, 72, 72, 64, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333140702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333155299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[1]R flush_numbers[0..12]: [128, 120, 118, 117, 126, 122, 123, 122, 122, 110, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333168956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[1]R generation[0..12]: [75, 70, 68, 68, 74, 72, 72, 72, 72, 64, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333182402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33319601Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[2]R flush_numbers[0..12]: [128, 120, 118, 117, 126, 122, 123, 122, 122, 110, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333213876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[2]R generation[0..12]: [75, 70, 68, 68, 74, 72, 72, 72, 72, 64, 72, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333227813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33324139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333259946Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333272773Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.33328541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333297968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"":"downstairs","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333310545Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333322112Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"0","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333334829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"1","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333347317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"client":"2","":"downstairs","session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333359824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333371632Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ad1a1994-52db-4ca7-b9a8-cf67fe2cdfb6 is now active with session: 8e85bf7b-df28-4594-8734-a149efd5e17a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333386518Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333398046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488,"session_id":"8e85bf7b-df28-4594-8734-a149efd5e17a"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:07.333412153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 197 len 32768 data: 4 4 5 5 5 8 8 7 02/30 Write block 101 len 12288 data: 7 8 7 03/30 Read block 272 len 12288 04/30 Write block 43 len 36864 data: 3 4 6 6 5 6 6 7 5 05/30 Write block 281 len 12288 data: 10 10 10 06/30 Flush 07/30 Write block 31 len 16384 data: 6 7 6 10 08/30 Read block 240 len 36864 09/30 Write block 426 len 4096 data: 5 10/30 Read block 418 len 8192 11/30 Write block 520 len 28672 data: 5 6 6 6 6 8 8 12/30 Write block 280 len 8192 data: 9 11 13/30 Write block 189 len 32768 data: 8 9 8 10 6 5 7 4 14/30 Read block 582 len 24576 15/30 Read block 1 len 4096 16/30 Read block 52 len 12288 17/30 Read block 251 len 20480 18/30 Flush 19/30 Write block 511 len 16384 data: 6 5 4 4 20/30 Read block 546 len 40960 21/30 Write block 436 len 4096 data: 4 22/30 Write block 304 len 40960 data: 8 7 6 5 6 5 8 8 8 9 23/30 Write block 166 len 4096 data: 6 24/30 Read block 321 len 12288 25/30 Write block 165 len 20480 data: 6 7 5 6 6 26/30 Write block 85 len 4096 data: 13 27/30 Write block 60 len 4096 data: 10 28/30 Read block 261 len 40960 29/30 Read block 118 len 36864 30/30 Write block 201 len 28672 data: 6 9 9 8 8 9 10 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:07.351677706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1488} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 128 128 128 F F F 1 020-039 76 76 76 130 130 130 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 68 68 68 117 117 117 T T T 4 080-099 74 74 74 126 126 126 T T T 5 100-119 76 76 76 129 129 129 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 T T T 9 180-199 76 76 76 130 130 130 F F F 10 200-219 76 76 76 129 129 129 T T T 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 70 70 70 120 120 120 T T T 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 76 76 76 130 130 130 T T T 22 440-459 74 74 74 126 126 126 F F F 23 460-479 74 74 74 126 126 126 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 75 75 75 128 128 128 T T T 26 520-539 76 76 76 130 130 130 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 74 74 74 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 76, Max flush: 130 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 128 128 128 F F F 1 020-039 76 76 76 130 130 130 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 F F F 4 080-099 77 77 77 131 131 131 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 72 72 72 123 123 123 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 76 76 76 130 130 130 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 77 77 77 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 74 74 74 126 126 126 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 77 77 77 131 131 131 F F F 26 520-539 76 76 76 130 130 130 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 74 74 74 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.604273889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.6503853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651220559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651268129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651287914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651304361Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} {"msg":"Crucible ec192a1f-8aa0-42de-87df-06c4958c2ecb has session id: 67f3a703-45cb-4143-bb4c-9924b96a951d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651876407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Upstairs opts: Upstairs UUID: ec192a1f-8aa0-42de-87df-06c4958c2ecb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651925706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Crucible stats registered with UUID: ec192a1f-8aa0-42de-87df-06c4958c2ecb","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.651949251Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:11.651970386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652430847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652464099Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} {"msg":"ec192a1f-8aa0-42de-87df-06c4958c2ecb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652519567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652546452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652569816Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652583844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65259822Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652611267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652624775Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65264724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652671414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652694Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48748)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652733641Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56749)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652761595Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63830)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.652791168Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.653474051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.6535249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.654199303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.654329875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.654697116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.654879686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.654902651Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655212484Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[0]R flush_numbers[0..12]: [128, 130, 129, 131, 131, 129, 123, 122, 131, 130, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655715425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[0]R generation[0..12]: [75, 76, 76, 77, 77, 76, 72, 72, 77, 76, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655744659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655762935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[1]R flush_numbers[0..12]: [128, 130, 129, 131, 131, 129, 123, 122, 131, 130, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655780241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[1]R generation[0..12]: [75, 76, 76, 77, 77, 76, 72, 72, 77, 76, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655796897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655813804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[2]R flush_numbers[0..12]: [128, 130, 129, 131, 131, 129, 123, 122, 131, 130, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65583028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"[2]R generation[0..12]: [75, 76, 76, 77, 77, 76, 72, 72, 77, 76, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655846747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655871351Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655886998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655902734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655918211Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655933658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"":"downstairs","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655949574Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65596744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"0","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655983857Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"1","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.655999913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"client":"2","":"downstairs","session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65601603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.656030787Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"ec192a1f-8aa0-42de-87df-06c4958c2ecb is now active with session: 67f3a703-45cb-4143-bb4c-9924b96a951d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.656046263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.65606059Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496,"session_id":"67f3a703-45cb-4143-bb4c-9924b96a951d"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:11.656075177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 32768 data: 4 5 5 5 6 6 7 6 02/30 Write block 489 len 36864 data: 5 4 4 4 2 5 4 6 7 03/30 Read block 268 len 16384 04/30 Read block 147 len 32768 05/30 Flush 06/30 Write block 565 len 16384 data: 8 9 10 8 07/30 Read block 579 len 28672 08/30 Read block 25 len 40960 09/30 Read block 57 len 36864 10/30 Read block 140 len 32768 11/30 Read block 519 len 32768 12/30 Write block 178 len 40960 data: 9 7 7 7 8 8 9 8 9 9 13/30 Read block 114 len 16384 14/30 Read block 562 len 28672 15/30 Read block 23 len 16384 16/30 Read block 134 len 12288 17/30 Write block 530 len 32768 data: 9 10 9 10 9 7 5 9 18/30 Read block 482 len 28672 19/30 Read block 473 len 32768 20/30 Write block 138 len 4096 data: 5 21/30 Read block 391 len 4096 22/30 Write block 89 len 28672 data: 8 6 6 4 4 3 3 23/30 Write block 470 len 12288 data: 8 10 10 24/30 Read block 207 len 12288 25/30 Read block 168 len 40960 26/30 Read block 580 len 32768 27/30 Write block 453 len 32768 data: 9 10 8 8 7 7 7 6 28/30 Write block 480 len 16384 data: 8 8 6 7 29/30 Read block 588 len 24576 30/30 Read block 122 len 4096 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:11.677133715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1496} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 132 132 132 F F F 1 020-039 76 76 76 130 130 130 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 F F F 4 080-099 77 77 77 131 131 131 T T T 5 100-119 76 76 76 129 129 129 F F F 6 120-139 72 72 72 123 123 123 T T T 7 140-159 72 72 72 122 122 122 F F F 8 160-179 77 77 77 131 131 131 T T T 9 180-199 76 76 76 130 130 130 T T T 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 77 77 77 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 74 74 74 126 126 126 T T T 23 460-479 74 74 74 126 126 126 T T T 24 480-499 78 78 78 132 132 132 T T T 25 500-519 77 77 77 131 131 131 F F F 26 520-539 76 76 76 130 130 130 T T T 27 540-559 72 72 72 123 123 123 F F F 28 560-579 74 74 74 126 126 126 T T T 29 580-599 64 64 64 110 110 110 F F F Max gen: 78, Max flush: 132 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 132 132 132 F F F 1 020-039 76 76 76 130 130 130 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 F F F 4 080-099 79 79 79 133 133 133 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 133 133 133 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 79 79 79 133 133 133 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 F F F 13 260-279 70 70 70 120 120 120 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 77 77 77 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 126 126 126 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 79 79 79 133 133 133 F F F 24 480-499 79 79 79 133 133 133 F F F 25 500-519 77 77 77 131 131 131 F F F 26 520-539 79 79 79 133 133 133 F F F 27 540-559 72 72 72 123 123 123 F F F 28 560-579 79 79 79 133 133 133 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 79, Max flush: 133 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.942561312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.986924762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.987814449Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.987856451Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.987871247Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.987885374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} {"msg":"Crucible 84c89512-17f6-4de4-a081-e199301348ba has session id: 349f0798-49db-4d88-b131-95ab89ad0d14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.988476516Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Upstairs opts: Upstairs UUID: 84c89512-17f6-4de4-a081-e199301348ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.988523776Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Crucible stats registered with UUID: 84c89512-17f6-4de4-a081-e199301348ba","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.988547791Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:25.988570416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989046143Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989077836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} {"msg":"84c89512-17f6-4de4-a081-e199301348ba active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989096742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989109349Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989131614Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989144971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989158529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989171835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989185572Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.98919866Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989212507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989226054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"ds_connection connected from Ok(127.0.0.1:44589)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989405025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:25.989446076Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:25.9895214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:25.989564661Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"ds_connection connected from Ok(127.0.0.1:44862)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989588825Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.989616379Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.990099405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.990130768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.990755292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.990872147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.991335257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:25.991467578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.990026223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"ds_connection connected from Ok(127.0.0.1:60421)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.990163054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.991208987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.991870544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[0]R flush_numbers[0..12]: [132, 130, 129, 131, 133, 129, 133, 122, 133, 133, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992513565Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[0]R generation[0..12]: [78, 76, 76, 77, 79, 76, 79, 72, 79, 79, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992543768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992560905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[1]R flush_numbers[0..12]: [132, 130, 129, 131, 133, 129, 133, 122, 133, 133, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992575691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[1]R generation[0..12]: [78, 76, 76, 77, 79, 76, 79, 72, 79, 79, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992589618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992604115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[2]R flush_numbers[0..12]: [132, 130, 129, 131, 133, 129, 133, 122, 133, 133, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992618212Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[2]R generation[0..12]: [78, 76, 76, 77, 79, 76, 79, 72, 79, 79, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992631949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992646376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992659803Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.9926736Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992686737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992706953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"":"downstairs","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.99272048Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992732587Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"0","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992748514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"1","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992762211Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"client":"2","":"downstairs","session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992775798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992805172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"84c89512-17f6-4de4-a081-e199301348ba is now active with session: 349f0798-49db-4d88-b131-95ab89ad0d14","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992818738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992830956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504,"session_id":"349f0798-49db-4d88-b131-95ab89ad0d14"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:35.992843523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 4096 data: 6 02/30 Write block 116 len 20480 data: 6 5 6 8 9 03/30 Write block 197 len 4096 data: 5 04/30 Read block 559 len 36864 05/30 Write block 354 len 8192 data: 5 5 06/30 Read block 95 len 24576 07/30 Read block 101 len 20480 08/30 Read block 101 len 28672 09/30 Write block 329 len 32768 data: 7 7 7 7 5 5 4 4 10/30 Write block 420 len 24576 data: 7 6 4 4 5 5 11/30 Flush 12/30 Write block 462 len 4096 data: 6 13/30 Write block 319 len 12288 data: 6 6 7 14/30 Flush 15/30 Write block 530 len 28672 data: 10 11 10 11 10 8 6 16/30 Read block 31 len 20480 17/30 Read block 316 len 8192 18/30 Write block 24 len 36864 data: 8 7 8 10 9 6 7 7 8 19/30 Read block 285 len 40960 20/30 Read block 92 len 8192 21/30 Write block 339 len 4096 data: 8 22/30 Write block 151 len 36864 data: 7 6 6 5 4 7 8 8 12 23/30 Read block 429 len 24576 24/30 Write block 264 len 8192 data: 8 8 25/30 Write block 545 len 12288 data: 9 9 6 26/30 Write block 544 len 4096 data: 12 27/30 Read block 472 len 32768 28/30 Write block 255 len 4096 data: 7 29/30 Read block 239 len 24576 30/30 Write block 504 len 8192 data: 9 10 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:36.011964251Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1504} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 132 132 132 F F F 1 020-039 76 76 76 130 130 130 T T T 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 F F F 4 080-099 79 79 79 133 133 133 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 72 72 72 122 122 122 T T T 8 160-179 79 79 79 133 133 133 F F F 9 180-199 80 80 80 134 134 134 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 74 74 74 126 126 126 T T T 13 260-279 70 70 70 120 120 120 T T T 14 280-299 76 76 76 130 130 130 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 80 80 80 135 135 135 T T T 17 340-359 80 80 80 134 134 134 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 80 80 80 134 134 134 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 79 79 79 133 133 133 F F F 25 500-519 77 77 77 131 131 131 T T T 26 520-539 79 79 79 133 133 133 T T T 27 540-559 72 72 72 123 123 123 T T T 28 560-579 79 79 79 133 133 133 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 80, Max flush: 135 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 132 132 132 F F F 1 020-039 81 81 81 136 136 136 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 F F F 4 080-099 79 79 79 133 133 133 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 80 80 80 134 134 134 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 81 81 81 136 136 136 F F F 13 260-279 81 81 81 136 136 136 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 F F F 17 340-359 80 80 80 134 134 134 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 67 67 67 113 113 113 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 80 80 80 134 134 134 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 79 79 79 133 133 133 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 81 81 81 136 136 136 F F F 27 540-559 81 81 81 136 136 136 F F F 28 560-579 79 79 79 133 133 133 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 81, Max flush: 136 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.268047969Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.311262747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.311932512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.311975543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.311990199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.312004576Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} {"msg":"Crucible 90cca5ba-88d4-429d-b7ee-46580fa2d6ba has session id: 5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.312537851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Upstairs opts: Upstairs UUID: 90cca5ba-88d4-429d-b7ee-46580fa2d6ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.312638319Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Crucible stats registered with UUID: 90cca5ba-88d4-429d-b7ee-46580fa2d6ba","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.312668843Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:40.312683419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31318838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313216014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} {"msg":"90cca5ba-88d4-429d-b7ee-46580fa2d6ba active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313285219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313303455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31332612Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313340427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313354604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313367741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313381378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313393655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313449394Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313475178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"ds_connection connected from Ok(127.0.0.1:43350)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31350863Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"ds_connection connected from Ok(127.0.0.1:38090)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313535195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"ds_connection connected from Ok(127.0.0.1:62500)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.313561749Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.314218767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.314398098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.314825235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.314969614Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.315287115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.315463467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.315489081Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.315813691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[0]R flush_numbers[0..12]: [132, 136, 129, 131, 133, 134, 134, 136, 133, 134, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316298016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[0]R generation[0..12]: [78, 81, 76, 77, 79, 80, 80, 81, 79, 80, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316318142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316333798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[1]R flush_numbers[0..12]: [132, 136, 129, 131, 133, 134, 134, 136, 133, 134, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316347926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[1]R generation[0..12]: [78, 81, 76, 77, 79, 80, 80, 81, 79, 80, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316361383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31637525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[2]R flush_numbers[0..12]: [132, 136, 129, 131, 133, 134, 134, 136, 133, 134, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316388817Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[2]R generation[0..12]: [78, 81, 76, 77, 79, 80, 80, 81, 79, 80, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316402154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31641904Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316437376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316450253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316462611Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316474888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"":"downstairs","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316487375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316499163Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"0","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.31651214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"1","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316524977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"client":"2","":"downstairs","session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316538064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316550082Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"90cca5ba-88d4-429d-b7ee-46580fa2d6ba is now active with session: 5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316562369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:41:40.316574177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512,"session_id":"5d7bbdfe-5bb8-4b2f-a5f5-b3ecfbd7ae34"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:40.316586894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 360 len 16384 data: 3 3 3 4 02/30 Read block 9 len 24576 03/30 Write block 244 len 24576 data: 5 4 3 2 2 2 04/30 Write block 219 len 4096 data: 5 05/30 Read block 304 len 28672 06/30 Flush 07/30 Write block 562 len 4096 data: 5 08/30 Read block 162 len 4096 09/30 Flush 10/30 Write block 5 len 36864 data: 7 8 7 8 7 9 10 11 11 11/30 Write block 61 len 28672 data: 11 11 8 6 6 8 7 12/30 Write block 67 len 32768 data: 8 6 7 6 6 7 7 5 13/30 Write block 264 len 8192 data: 9 9 14/30 Write block 78 len 12288 data: 3 3 6 15/30 Read block 182 len 4096 16/30 Write block 426 len 12288 data: 6 7 5 17/30 Read block 64 len 16384 18/30 Read block 392 len 16384 19/30 Write block 559 len 16384 data: 8 7 6 6 20/30 Write block 575 len 12288 data: 6 6 6 21/30 Read block 262 len 8192 22/30 Read block 479 len 20480 23/30 Write block 399 len 12288 data: 7 7 8 24/30 Write block 204 len 16384 data: 9 9 10 11 25/30 Write block 488 len 36864 data: 5 6 5 5 5 3 6 5 7 26/30 Read block 123 len 40960 27/30 Write block 237 len 24576 data: 4 6 6 7 6 8 28/30 Write block 2 len 28672 data: 6 6 7 8 9 8 9 29/30 Read block 66 len 4096 30/30 Read block 478 len 32768 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 9 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 21 21 20 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:40.334314603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1512} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 132 132 132 T T T 1 020-039 81 81 81 136 136 136 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 77 77 77 131 131 131 T T T 4 080-099 79 79 79 133 133 133 T T T 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 80 80 80 134 134 134 F F F 10 200-219 82 82 82 137 137 137 T T T 11 220-239 75 75 75 128 128 128 T T T 12 240-259 82 82 82 137 137 137 T T T 13 260-279 81 81 81 136 136 136 T T T 14 280-299 76 76 76 130 130 130 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 F F F 17 340-359 80 80 80 134 134 134 F F F 18 360-379 82 82 82 137 137 137 F F F 19 380-399 67 67 67 113 113 113 T T T 20 400-419 64 64 64 110 110 110 T T T 21 420-439 80 80 80 134 134 134 T T T 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 79 79 79 133 133 133 T T T 25 500-519 81 81 81 136 136 136 F F F 26 520-539 81 81 81 136 136 136 F F F 27 540-559 81 81 81 136 136 136 T T T 28 560-579 82 82 82 138 138 138 T T T 29 580-599 64 64 64 110 110 110 F F F Max gen: 82, Max flush: 138 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 139 139 139 F F F 1 020-039 81 81 81 136 136 136 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 83 83 83 139 139 139 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 80 80 80 134 134 134 F F F 10 200-219 83 83 83 139 139 139 F F F 11 220-239 83 83 83 139 139 139 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 83 83 83 139 139 139 F F F 14 280-299 76 76 76 130 130 130 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 F F F 17 340-359 80 80 80 134 134 134 F F F 18 360-379 82 82 82 137 137 137 F F F 19 380-399 83 83 83 139 139 139 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 83 83 83 139 139 139 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 83 83 83 139 139 139 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 81 81 81 136 136 136 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 83 83 83 139 139 139 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 83, Max flush: 139 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.615430958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.659342056Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.660041834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.660086364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66010594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.660121687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} {"msg":"Crucible bad7681d-b0ac-4f1f-bdb2-1e38245413b0 has session id: fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.660700481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Upstairs opts: Upstairs UUID: bad7681d-b0ac-4f1f-bdb2-1e38245413b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.6607511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Crucible stats registered with UUID: bad7681d-b0ac-4f1f-bdb2-1e38245413b0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.660776285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:44.660791042Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661284695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661309989Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} {"msg":"bad7681d-b0ac-4f1f-bdb2-1e38245413b0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661376895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661394171Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661416676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661430913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66144563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661460067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661484851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661515695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661540689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661555626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"ds_connection connected from Ok(127.0.0.1:53517)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661569133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"ds_connection connected from Ok(127.0.0.1:34231)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.661683818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"ds_connection connected from Ok(127.0.0.1:57722)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66172348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.662358062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.662387236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.662954343Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.663091163Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.663133124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.663511792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.663684405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.663762528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[0]R flush_numbers[0..12]: [139, 136, 129, 139, 139, 134, 134, 136, 133, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664241194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[0]R generation[0..12]: [83, 81, 76, 83, 83, 80, 80, 81, 79, 80, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664268808Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664284795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[1]R flush_numbers[0..12]: [139, 136, 129, 139, 139, 134, 134, 136, 133, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664298582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[1]R generation[0..12]: [83, 81, 76, 83, 83, 80, 80, 81, 79, 80, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664311889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664325346Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[2]R flush_numbers[0..12]: [139, 136, 129, 139, 139, 134, 134, 136, 133, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664338923Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[2]R generation[0..12]: [83, 81, 76, 83, 83, 80, 80, 81, 79, 80, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66435241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664367007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664386053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66439895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664411388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664423935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"":"downstairs","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664436642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664449249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"0","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664465166Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"1","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664478323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"client":"2","":"downstairs","session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66449134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664503208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"bad7681d-b0ac-4f1f-bdb2-1e38245413b0 is now active with session: fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664515685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.664527272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520,"session_id":"fdc70b80-9c9d-4c9f-8d3d-27f2cfc9d351"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:44.66453906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 20480 data: 10 7 8 9 7 02/30 Write block 435 len 12288 data: 3 5 5 03/30 Read block 402 len 16384 04/30 Read block 561 len 32768 05/30 Read block 137 len 24576 06/30 Write block 580 len 28672 data: 7 7 7 4 4 3 4 07/30 Read block 376 len 32768 08/30 Write block 363 len 40960 data: 5 4 4 4 5 5 4 5 5 5 09/30 Write block 190 len 32768 data: 10 9 11 7 6 8 5 6 10/30 Write block 357 len 16384 data: 4 3 3 4 11/30 Write block 286 len 12288 data: 7 7 7 12/30 Read block 445 len 28672 13/30 Read block 464 len 36864 14/30 Read block 460 len 32768 15/30 Read block 529 len 16384 16/30 Read block 331 len 4096 17/30 Read block 415 len 40960 18/30 Write block 575 len 4096 data: 7 19/30 Read block 357 len 16384 20/30 Write block 60 len 12288 data: 11 12 12 21/30 Read block 309 len 16384 22/30 Write block 477 len 20480 data: 6 6 6 9 9 23/30 Read block 503 len 24576 24/30 Write block 185 len 40960 data: 9 10 10 10 9 11 10 12 8 7 25/30 Read block 317 len 32768 26/30 Read block 74 len 20480 27/30 Write block 541 len 36864 data: 14 13 12 13 10 10 7 8 6 28/30 Read block 503 len 28672 29/30 Write block 370 len 16384 data: 6 6 6 6 30/30 Write block 537 len 40960 data: 10 8 13 14 15 14 13 14 11 11 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:44.685554148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1520} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 139 139 139 F F F 1 020-039 81 81 81 136 136 136 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 83 83 83 139 139 139 T T T 4 080-099 83 83 83 139 139 139 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 80 80 80 134 134 134 T T T 10 200-219 83 83 83 139 139 139 F F F 11 220-239 83 83 83 139 139 139 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 83 83 83 139 139 139 T T T 14 280-299 76 76 76 130 130 130 T T T 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 F F F 17 340-359 80 80 80 134 134 134 T T T 18 360-379 82 82 82 137 137 137 T T T 19 380-399 83 83 83 139 139 139 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 83 83 83 139 139 139 T T T 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 135 135 135 T T T 24 480-499 83 83 83 139 139 139 T T T 25 500-519 81 81 81 136 136 136 F F F 26 520-539 81 81 81 136 136 136 T T T 27 540-559 83 83 83 139 139 139 T T T 28 560-579 83 83 83 139 139 139 T T T 29 580-599 64 64 64 110 110 110 T T T Max gen: 83, Max flush: 139 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 139 139 139 F F F 1 020-039 81 81 81 136 136 136 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 84 84 84 140 140 140 F F F 10 200-219 83 83 83 139 139 139 F F F 11 220-239 83 83 83 139 139 139 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 84 84 84 140 140 140 F F F 14 280-299 84 84 84 140 140 140 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 F F F 17 340-359 84 84 84 140 140 140 F F F 18 360-379 84 84 84 140 140 140 F F F 19 380-399 83 83 83 139 139 139 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 140 140 140 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 84 84 84 140 140 140 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 84 84 84 140 140 140 F F F 27 540-559 84 84 84 140 140 140 F F F 28 560-579 84 84 84 140 140 140 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 84, Max flush: 140 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:58.969665475Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:59.013003157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.013537261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.045308959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046128252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046173402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046189918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046204785Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} {"msg":"Crucible fc433e25-6ddc-4cf0-9955-87f8dd64c16d has session id: 61dcf9ab-b193-45e4-85fd-3e9751637070","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046771442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Upstairs opts: Upstairs UUID: fc433e25-6ddc-4cf0-9955-87f8dd64c16d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046813813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Crucible stats registered with UUID: fc433e25-6ddc-4cf0-9955-87f8dd64c16d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.046838278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:59.046853374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047439917Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047464752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} {"msg":"fc433e25-6ddc-4cf0-9955-87f8dd64c16d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047530158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047548364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047573199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047587305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047601482Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.04761423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047628207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047641424Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047655391Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047668958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"ds_connection connected from Ok(127.0.0.1:44839)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047806448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"ds_connection connected from Ok(127.0.0.1:47788)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.047846579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"ds_connection connected from Ok(127.0.0.1:50203)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.048015673Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.048613123Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.048651375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.048715671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.049254054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.049397873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.049428397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.049761514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050008761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[0]R flush_numbers[0..12]: [139, 136, 129, 140, 139, 134, 134, 136, 133, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050080465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[0]R generation[0..12]: [83, 81, 76, 84, 83, 80, 80, 81, 79, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050100151Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050115138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[1]R flush_numbers[0..12]: [139, 136, 129, 140, 139, 134, 134, 136, 133, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050128855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[1]R generation[0..12]: [83, 81, 76, 84, 83, 80, 80, 81, 79, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050145511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050159288Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[2]R flush_numbers[0..12]: [139, 136, 129, 140, 139, 134, 134, 136, 133, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050172565Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[2]R generation[0..12]: [83, 81, 76, 84, 83, 80, 80, 81, 79, 84, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050185393Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.05019875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Max found gen is 85","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050211247Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Generation requested: 86 >= found:85","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050224114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:41:59.050236662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050256847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"":"downstairs","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050269864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050281642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"0","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050294819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"1","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050307656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"client":"2","":"downstairs","session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050320283Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.05033389Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"fc433e25-6ddc-4cf0-9955-87f8dd64c16d is now active with session: 61dcf9ab-b193-45e4-85fd-3e9751637070","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050346708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050358335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528,"session_id":"61dcf9ab-b193-45e4-85fd-3e9751637070"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:41:59.050370102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 12288 data: 7 6 7 02/30 Read block 568 len 20480 03/30 Write block 275 len 12288 data: 8 8 8 04/30 Flush 05/30 Write block 528 len 8192 data: 7 10 06/30 Read block 392 len 32768 07/30 Read block 75 len 32768 08/30 Read block 402 len 36864 09/30 Read block 293 len 4096 10/30 Read block 559 len 16384 11/30 Read block 201 len 4096 12/30 Write block 554 len 16384 data: 7 7 8 8 13/30 Read block 295 len 32768 14/30 Read block 206 len 4096 15/30 Read block 395 len 12288 16/30 Write block 453 len 16384 data: 10 11 9 9 17/30 Write block 231 len 20480 data: 6 7 6 5 6 18/30 Read block 152 len 8192 19/30 Read block 567 len 20480 20/30 Read block 199 len 28672 21/30 Read block 353 len 16384 22/30 Read block 291 len 24576 23/30 Write block 189 len 4096 data: 10 24/30 Write block 335 len 8192 data: 5 5 25/30 Write block 324 len 20480 data: 7 7 5 5 7 26/30 Read block 41 len 32768 27/30 Read block 70 len 20480 28/30 Write block 460 len 36864 data: 7 6 7 5 5 5 7 9 9 29/30 Write block 350 len 16384 data: 7 6 6 7 30/30 Write block 328 len 28672 data: 8 8 8 8 8 6 6 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:41:59.069944282Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1528} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 86 86 86 141 141 141 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 84 84 84 140 140 140 T T T 10 200-219 83 83 83 139 139 139 F F F 11 220-239 83 83 83 139 139 139 T T T 12 240-259 83 83 83 139 139 139 F F F 13 260-279 86 86 86 141 141 141 F F F 14 280-299 84 84 84 140 140 140 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 81 81 81 136 136 136 T T T 17 340-359 84 84 84 140 140 140 T T T 18 360-379 84 84 84 140 140 140 F F F 19 380-399 83 83 83 139 139 139 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 140 140 140 F F F 22 440-459 79 79 79 133 133 133 T T T 23 460-479 84 84 84 140 140 140 T T T 24 480-499 84 84 84 140 140 140 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 84 84 84 140 140 140 T T T 27 540-559 84 84 84 140 140 140 T T T 28 560-579 84 84 84 140 140 140 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 86, Max flush: 141 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 86 86 86 141 141 141 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 80 80 80 134 134 134 F F F 6 120-139 80 80 80 134 134 134 F F F 7 140-159 81 81 81 136 136 136 F F F 8 160-179 79 79 79 133 133 133 F F F 9 180-199 87 87 87 142 142 142 F F F 10 200-219 83 83 83 139 139 139 F F F 11 220-239 87 87 87 142 142 142 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 86 86 86 141 141 141 F F F 14 280-299 84 84 84 140 140 140 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 87 87 87 142 142 142 F F F 18 360-379 84 84 84 140 140 140 F F F 19 380-399 83 83 83 139 139 139 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 140 140 140 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 87 87 87 142 142 142 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 87 87 87 142 142 142 F F F 27 540-559 87 87 87 142 142 142 F F F 28 560-579 84 84 84 140 140 140 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 87, Max flush: 142 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.365588219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.408476818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409149742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409200191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409218018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409235514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} {"msg":"Crucible 99bd9c5d-9534-41c2-b4ce-c2a6d9dca694 has session id: 0de51b92-6065-463d-82c9-74ebe2f83c1c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409729947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Upstairs opts: Upstairs UUID: 99bd9c5d-9534-41c2-b4ce-c2a6d9dca694, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409772947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Crucible stats registered with UUID: 99bd9c5d-9534-41c2-b4ce-c2a6d9dca694","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.409789154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:03.409802341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.4102658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410291685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} {"msg":"99bd9c5d-9534-41c2-b4ce-c2a6d9dca694 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410383055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.41040688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410433724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410448431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410469016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410491991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410514077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410535982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410558597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410581742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37105)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410623313Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55381)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410649237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38617)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.410672242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.411367822Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.411396906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.412059722Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.412103073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.412702463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.412823716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.412927414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413397702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 134, 134, 136, 133, 142, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.41387002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[0]R generation[0..12]: [86, 86, 76, 84, 83, 80, 80, 81, 79, 87, 83, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413891425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413906962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 134, 134, 136, 133, 142, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413920839Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[1]R generation[0..12]: [86, 86, 76, 84, 83, 80, 80, 81, 79, 87, 83, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413934256Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413948962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 134, 134, 136, 133, 142, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.4139624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[2]R generation[0..12]: [86, 86, 76, 84, 83, 80, 80, 81, 79, 87, 83, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413975607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.413989214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:42:03.41400626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414022427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414035124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414048561Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"":"downstairs","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414061658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414073366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"0","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414086453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"1","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.41409913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"client":"2","":"downstairs","session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414111987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414123685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"99bd9c5d-9534-41c2-b4ce-c2a6d9dca694 is now active with session: 0de51b92-6065-463d-82c9-74ebe2f83c1c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414136182Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414147639Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536,"session_id":"0de51b92-6065-463d-82c9-74ebe2f83c1c"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:03.414159457Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 105 len 12288 data: 7 7 7 03/30 Write block 233 len 28672 data: 7 6 7 4 5 7 7 04/30 Read block 293 len 40960 05/30 Read block 545 len 36864 06/30 Write block 123 len 12288 data: 8 8 6 07/30 Read block 516 len 32768 08/30 Write block 379 len 24576 data: 6 9 7 8 9 9 09/30 Read block 327 len 40960 10/30 Read block 463 len 16384 11/30 Flush 12/30 Write block 577 len 32768 data: 7 7 8 8 8 8 5 5 13/30 Write block 375 len 20480 data: 6 6 4 4 7 14/30 Read block 527 len 4096 15/30 Write block 157 len 24576 data: 9 9 13 9 9 9 16/30 Read block 191 len 16384 17/30 Flush 18/30 Flush 19/30 Write block 476 len 32768 data: 6 7 7 7 10 10 7 8 20/30 Read block 93 len 40960 21/30 Read block 466 len 32768 22/30 Write block 344 len 8192 data: 8 9 23/30 Read block 380 len 8192 24/30 Read block 322 len 20480 25/30 Read block 272 len 28672 26/30 Read block 278 len 12288 27/30 Write block 280 len 32768 data: 10 12 11 11 9 8 8 8 28/30 Write block 156 len 28672 data: 8 10 10 14 10 10 10 29/30 Write block 135 len 32768 data: 10 9 6 6 9 9 6 7 30/30 Read block 437 len 36864 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:03.435506023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1536} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 86 86 86 141 141 141 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 88 88 88 144 144 144 F F F 6 120-139 88 88 88 144 144 144 T T T 7 140-159 88 88 88 145 145 145 T T T 8 160-179 88 88 88 145 145 145 T T T 9 180-199 87 87 87 142 142 142 F F F 10 200-219 83 83 83 139 139 139 F F F 11 220-239 88 88 88 144 144 144 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 86 86 86 141 141 141 F F F 14 280-299 84 84 84 140 140 140 T T T 15 300-319 80 80 80 135 135 135 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 87 87 87 142 142 142 T T T 18 360-379 88 88 88 145 145 145 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 140 140 140 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 87 87 87 142 142 142 T T T 24 480-499 84 84 84 140 140 140 T T T 25 500-519 81 81 81 136 136 136 F F F 26 520-539 87 87 87 142 142 142 F F F 27 540-559 87 87 87 142 142 142 F F F 28 560-579 88 88 88 145 145 145 F F F 29 580-599 88 88 88 145 145 145 F F F Max gen: 88, Max flush: 145 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 86 86 86 141 141 141 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 88 88 88 144 144 144 F F F 6 120-139 89 89 89 146 146 146 F F F 7 140-159 89 89 89 146 146 146 F F F 8 160-179 89 89 89 146 146 146 F F F 9 180-199 87 87 87 142 142 142 F F F 10 200-219 83 83 83 139 139 139 F F F 11 220-239 88 88 88 144 144 144 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 86 86 86 141 141 141 F F F 14 280-299 89 89 89 146 146 146 F F F 15 300-319 80 80 80 135 135 135 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 89 89 89 146 146 146 F F F 18 360-379 88 88 88 145 145 145 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 140 140 140 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 89 89 89 146 146 146 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 87 87 87 142 142 142 F F F 27 540-559 87 87 87 142 142 142 F F F 28 560-579 88 88 88 145 145 145 F F F 29 580-599 88 88 88 145 145 145 F F F Max gen: 89, Max flush: 146 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.694643388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:07.737453114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.737903277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.769449323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770124917Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770160749Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770175666Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770190712Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} {"msg":"Crucible 34066b04-e4f4-4430-859b-25176963a2e9 has session id: 41c4310b-0b6a-4f42-8004-742509560c2b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.77089376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Upstairs opts: Upstairs UUID: 34066b04-e4f4-4430-859b-25176963a2e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770938701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Crucible stats registered with UUID: 34066b04-e4f4-4430-859b-25176963a2e9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.770962475Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:07.770983061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771482843Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771508797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} {"msg":"34066b04-e4f4-4430-859b-25176963a2e9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771577252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771594808Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771617443Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771640978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771666883Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771742606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.77177117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771795875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.7718193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771848233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49191)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.771874758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56799)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.772009278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44546)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.772046041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.772587433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.772622356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.772641781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773260068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773288352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773309207Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773906158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773933392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 144, 146, 146, 146, 142, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773956037Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[0]R generation[0..12]: [86, 86, 76, 84, 83, 88, 89, 89, 89, 87, 83, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773971653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.773990269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 144, 146, 146, 146, 142, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774005696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[1]R generation[0..12]: [86, 86, 76, 84, 83, 88, 89, 89, 89, 87, 83, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774019373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.77403322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 129, 140, 139, 144, 146, 146, 146, 142, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774047787Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[2]R generation[0..12]: [86, 86, 76, 84, 83, 88, 89, 89, 89, 87, 83, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774061244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774074921Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774087828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774100765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:42:07.774113513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774133228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"":"downstairs","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774146405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774158413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"0","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.77417167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"1","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774185317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"client":"2","":"downstairs","session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774201653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774214161Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"34066b04-e4f4-4430-859b-25176963a2e9 is now active with session: 41c4310b-0b6a-4f42-8004-742509560c2b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774226678Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774238315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544,"session_id":"41c4310b-0b6a-4f42-8004-742509560c2b"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:07.774250223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 24576 data: 7 8 5 6 8 8 02/30 Read block 257 len 40960 03/30 Read block 420 len 24576 04/30 Write block 417 len 32768 data: 6 7 6 8 7 5 5 6 05/30 Write block 299 len 16384 data: 3 4 6 7 06/30 Read block 376 len 20480 07/30 Read block 530 len 24576 08/30 Read block 512 len 32768 09/30 Flush 10/30 Write block 286 len 36864 data: 9 9 8 7 6 8 7 6 8 11/30 Read block 184 len 12288 12/30 Flush 13/30 Write block 178 len 20480 data: 10 8 8 8 9 14/30 Write block 300 len 24576 data: 5 7 8 8 9 8 15/30 Read block 427 len 24576 16/30 Read block 305 len 16384 17/30 Flush 18/30 Write block 266 len 36864 data: 8 9 10 8 5 6 7 7 7 19/30 Read block 374 len 12288 20/30 Read block 444 len 20480 21/30 Write block 62 len 20480 data: 13 9 7 7 9 22/30 Read block 536 len 36864 23/30 Read block 391 len 16384 24/30 Write block 407 len 36864 data: 6 7 8 7 8 8 5 7 6 25/30 Write block 418 len 40960 data: 8 7 9 8 6 6 7 6 7 8 26/30 Write block 29 len 4096 data: 7 27/30 Write block 199 len 32768 data: 6 6 7 10 10 10 10 11 28/30 Write block 155 len 24576 data: 5 9 11 11 15 11 29/30 Read block 479 len 40960 30/30 Write block 28 len 36864 data: 10 8 8 8 9 7 11 8 8 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:07.796228022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1544} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 86 86 86 141 141 141 T T T 2 040-059 76 76 76 129 129 129 F F F 3 060-079 84 84 84 140 140 140 T T T 4 080-099 83 83 83 139 139 139 F F F 5 100-119 88 88 88 144 144 144 F F F 6 120-139 89 89 89 146 146 146 F F F 7 140-159 89 89 89 146 146 146 T T T 8 160-179 90 90 90 149 149 149 T T T 9 180-199 90 90 90 149 149 149 T T T 10 200-219 83 83 83 139 139 139 T T T 11 220-239 90 90 90 147 147 147 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 86 86 86 141 141 141 T T T 14 280-299 90 90 90 148 148 148 F F F 15 300-319 90 90 90 149 149 149 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 89 89 89 146 146 146 F F F 18 360-379 88 88 88 145 145 145 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 90 90 90 147 147 147 T T T 21 420-439 90 90 90 147 147 147 T T T 22 440-459 87 87 87 142 142 142 F F F 23 460-479 89 89 89 146 146 146 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 87 87 87 142 142 142 F F F 27 540-559 87 87 87 142 142 142 F F F 28 560-579 88 88 88 145 145 145 F F F 29 580-599 88 88 88 145 145 145 F F F Max gen: 90, Max flush: 149 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 91 91 91 150 150 150 F F F 2 040-059 76 76 76 129 129 129 F F F 3 060-079 91 91 91 150 150 150 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 88 88 88 144 144 144 F F F 6 120-139 89 89 89 146 146 146 F F F 7 140-159 91 91 91 150 150 150 F F F 8 160-179 91 91 91 150 150 150 F F F 9 180-199 91 91 91 150 150 150 F F F 10 200-219 91 91 91 150 150 150 F F F 11 220-239 90 90 90 147 147 147 F F F 12 240-259 83 83 83 139 139 139 F F F 13 260-279 91 91 91 150 150 150 F F F 14 280-299 90 90 90 148 148 148 F F F 15 300-319 90 90 90 149 149 149 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 89 89 89 146 146 146 F F F 18 360-379 88 88 88 145 145 145 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 91 91 91 150 150 150 F F F 21 420-439 91 91 91 150 150 150 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 89 89 89 146 146 146 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 87 87 87 142 142 142 F F F 27 540-559 87 87 87 142 142 142 F F F 28 560-579 88 88 88 145 145 145 F F F 29 580-599 88 88 88 145 145 145 F F F Max gen: 91, Max flush: 150 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.098127454Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:12.141398331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.14204764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.173685846Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.174389234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.174424117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.174437934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17445214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} {"msg":"Crucible f51c5d20-892e-4c91-8588-c572b052cf50 has session id: 50f6c09a-9b8c-41c3-9dc2-adecc43820cb","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175079655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Upstairs opts: Upstairs UUID: f51c5d20-892e-4c91-8588-c572b052cf50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175123505Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Crucible stats registered with UUID: f51c5d20-892e-4c91-8588-c572b052cf50","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17514691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:12.175168176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175590524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175617308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} {"msg":"f51c5d20-892e-4c91-8588-c572b052cf50 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175678015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17570124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175725235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175739582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175756388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175780533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175807177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175832752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175848598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175868494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:42545)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17593305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48347)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.175970272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:50401)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.176002405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.176665931Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.176696944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.176905599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.177436664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.177474376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.177504969Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178004051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178132893Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[0]R flush_numbers[0..12]: [141, 150, 129, 150, 139, 144, 146, 150, 150, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17819408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[0]R generation[0..12]: [86, 91, 76, 91, 83, 88, 89, 91, 91, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178222764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17823914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[1]R flush_numbers[0..12]: [141, 150, 129, 150, 139, 144, 146, 150, 150, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178253537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[1]R generation[0..12]: [86, 91, 76, 91, 83, 88, 89, 91, 91, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178267144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178280751Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[2]R flush_numbers[0..12]: [141, 150, 129, 150, 139, 144, 146, 150, 150, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178294208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[2]R generation[0..12]: [86, 91, 76, 91, 83, 88, 89, 91, 91, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178307196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178323952Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178337139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-23T17:42:12.178349846Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178370732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178383519Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"":"downstairs","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178396556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178408374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"0","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17842238Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"1","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178435018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"client":"2","":"downstairs","session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178448185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178459982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"f51c5d20-892e-4c91-8588-c572b052cf50 is now active with session: 50f6c09a-9b8c-41c3-9dc2-adecc43820cb","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.17847231Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178483827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552,"session_id":"50f6c09a-9b8c-41c3-9dc2-adecc43820cb"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:12.178495695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 40960 data: 9 8 9 9 6 8 7 5 7 9 02/30 Write block 69 len 4096 data: 8 03/30 Write block 224 len 32768 data: 6 4 6 5 6 6 5 7 04/30 Read block 110 len 32768 05/30 Read block 561 len 12288 06/30 Write block 516 len 32768 data: 4 5 2 4 6 7 7 7 07/30 Write block 118 len 20480 data: 7 9 10 11 10 08/30 Read block 24 len 16384 09/30 Read block 43 len 36864 10/30 Read block 499 len 12288 11/30 Write block 358 len 36864 data: 4 4 5 4 4 6 5 5 5 12/30 Write block 144 len 16384 data: 6 7 8 8 13/30 Flush 14/30 Write block 186 len 8192 data: 11 11 15/30 Read block 265 len 20480 16/30 Write block 238 len 40960 data: 9 9 8 7 9 7 6 5 4 3 17/30 Flush 18/30 Write block 51 len 12288 data: 6 6 5 19/30 Write block 557 len 40960 data: 9 8 9 8 7 7 5 6 9 10 20/30 Read block 544 len 16384 21/30 Read block 266 len 24576 22/30 Read block 497 len 16384 23/30 Read block 332 len 12288 24/30 Write block 205 len 24576 data: 11 12 12 10 6 7 25/30 Write block 291 len 4096 data: 9 26/30 Read block 399 len 40960 27/30 Write block 541 len 32768 data: 16 15 14 15 12 12 8 9 28/30 Write block 140 len 20480 data: 10 7 8 6 7 29/30 Write block 141 len 36864 data: 8 9 7 8 8 9 9 7 8 30/30 Write block 370 len 32768 data: 7 7 7 7 5 7 7 5 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:12.198948914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1552} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 91 91 91 150 150 150 F F F 2 040-059 76 76 76 129 129 129 T T T 3 060-079 92 92 92 151 151 151 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 92 92 92 151 151 151 F F F 6 120-139 92 92 92 151 151 151 F F F 7 140-159 92 92 92 151 151 151 T T T 8 160-179 91 91 91 150 150 150 F F F 9 180-199 92 92 92 152 152 152 F F F 10 200-219 91 91 91 150 150 150 T T T 11 220-239 92 92 92 152 152 152 F F F 12 240-259 92 92 92 152 152 152 F F F 13 260-279 91 91 91 150 150 150 F F F 14 280-299 90 90 90 148 148 148 T T T 15 300-319 90 90 90 149 149 149 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 92 92 92 151 151 151 F F F 18 360-379 92 92 92 151 151 151 T T T 19 380-399 88 88 88 144 144 144 F F F 20 400-419 92 92 92 151 151 151 F F F 21 420-439 91 91 91 150 150 150 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 89 89 89 146 146 146 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 87 87 87 142 142 142 T T T 28 560-579 88 88 88 145 145 145 T T T 29 580-599 88 88 88 145 145 145 F F F Max gen: 92, Max flush: 152 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 91 91 91 150 150 150 F F F 2 040-059 93 93 93 153 153 153 F F F 3 060-079 92 92 92 151 151 151 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 92 92 92 151 151 151 F F F 6 120-139 92 92 92 151 151 151 F F F 7 140-159 93 93 93 153 153 153 F F F 8 160-179 91 91 91 150 150 150 F F F 9 180-199 92 92 92 152 152 152 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 92 92 92 152 152 152 F F F 12 240-259 92 92 92 152 152 152 F F F 13 260-279 91 91 91 150 150 150 F F F 14 280-299 93 93 93 153 153 153 F F F 15 300-319 90 90 90 149 149 149 F F F 16 320-339 87 87 87 142 142 142 F F F 17 340-359 92 92 92 151 151 151 F F F 18 360-379 93 93 93 153 153 153 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 92 92 92 151 151 151 F F F 21 420-439 91 91 91 150 150 150 F F F 22 440-459 87 87 87 142 142 142 F F F 23 460-479 89 89 89 146 146 146 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 93 93 93 153 153 153 F F F 28 560-579 93 93 93 153 153 153 F F F 29 580-599 88 88 88 145 145 145 F F F Max gen: 93, Max flush: 153 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.563027847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:16.606138939Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.606659786Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.637587126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.638339403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.638374836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.638393401Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.638408768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} {"msg":"Crucible 7fae4266-3516-46d9-bb64-3f12c9b86ee7 has session id: 2c08f54c-9d33-424a-8afc-49c70d6fe321","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.638983794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Upstairs opts: Upstairs UUID: 7fae4266-3516-46d9-bb64-3f12c9b86ee7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639034623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Crucible stats registered with UUID: 7fae4266-3516-46d9-bb64-3f12c9b86ee7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639058717Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:16.639074324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639447113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639470978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} {"msg":"7fae4266-3516-46d9-bb64-3f12c9b86ee7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639539653Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639556849Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639580015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639603579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639630014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639657518Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639686121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639718844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639749878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639775652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"ds_connection connected from Ok(127.0.0.1:39725)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639801647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"ds_connection connected from Ok(127.0.0.1:40749)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639828801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"ds_connection connected from Ok(127.0.0.1:50321)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.639992185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.640519261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.640559732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.640676237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.64121539Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641251262Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641277827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641891244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641915978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[0]R flush_numbers[0..12]: [141, 150, 153, 151, 139, 151, 151, 153, 150, 152, 153, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641972726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[0]R generation[0..12]: [86, 91, 93, 92, 83, 92, 92, 93, 91, 92, 93, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.641996901Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642011718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[1]R flush_numbers[0..12]: [141, 150, 153, 151, 139, 151, 151, 153, 150, 152, 153, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642025475Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[1]R generation[0..12]: [86, 91, 93, 92, 83, 92, 92, 93, 91, 92, 93, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642038612Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642052189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[2]R flush_numbers[0..12]: [141, 150, 153, 151, 139, 151, 151, 153, 150, 152, 153, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642065436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[2]R generation[0..12]: [86, 91, 93, 92, 83, 92, 92, 93, 91, 92, 93, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642078643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.64209185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642104458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642117055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642133541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX"Wait for a query_work_queue command to finish before sending IO ,"pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642167274Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"":"downstairs","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642193188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642209975Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"0","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642226331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"1","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642244387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"client":"2","":"downstairs","session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642268432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642292197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"7fae4266-3516-46d9-bb64-3f12c9b86ee7 is now active with session: 2c08f54c-9d33-424a-8afc-49c70d6fe321","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642313762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642326229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560,"session_id":"2c08f54c-9d33-424a-8afc-49c70d6fe321"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:16.642341576Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 36864 data: 9 10 11 9 6 7 8 8 8 02/30 Read block 577 len 20480 03/30 Read block 223 len 20480 04/30 Flush 05/30 Write block 31 len 4096 data: 9 06/30 Read block 435 len 36864 07/30 Write block 73 len 28672 data: 8 6 5 3 3 4 4 08/30 Write block 235 len 36864 data: 9 6 7 10 10 9 8 10 8 09/30 Read block 596 len 4096 10/30 Read block 400 len 4096 11/30 Read block 297 len 16384 12/30 Read block 471 len 32768 13/30 Read block 248 len 24576 14/30 Read block 383 len 40960 15/30 Read block 342 len 36864 16/30 Write block 23 len 24576 data: 8 9 8 9 11 11 17/30 Write block 577 len 20480 data: 8 8 9 9 9 18/30 Write block 167 len 24576 data: 6 7 7 6 8 8 19/30 Read block 532 len 12288 20/30 Flush 21/30 Write block 456 len 28672 data: 10 8 8 8 8 7 8 22/30 Read block 419 len 4096 23/30 Read block 203 len 24576 24/30 Flush 25/30 Write block 122 len 36864 data: 11 9 9 7 3 4 5 5 4 26/30 Write block 347 len 16384 data: 7 6 7 8 27/30 Write block 319 len 24576 data: 7 7 8 5 6 8 28/30 Read block 417 len 4096 29/30 Write block 424 len 20480 data: 8 7 8 9 6 30/30 Read block 172 len 32768 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:16.66291169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1560} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 94 94 94 155 155 155 F F F 2 040-059 93 93 93 153 153 153 F F F 3 060-079 94 94 94 155 155 155 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 92 92 92 151 151 151 F F F 6 120-139 92 92 92 151 151 151 T T T 7 140-159 93 93 93 153 153 153 F F F 8 160-179 94 94 94 155 155 155 F F F 9 180-199 92 92 92 152 152 152 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 94 94 94 155 155 155 F F F 13 260-279 94 94 94 154 154 154 F F F 14 280-299 93 93 93 153 153 153 F F F 15 300-319 90 90 90 149 149 149 T T T 16 320-339 87 87 87 142 142 142 T T T 17 340-359 92 92 92 151 151 151 T T T 18 360-379 93 93 93 153 153 153 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 92 92 92 151 151 151 F F F 21 420-439 91 91 91 150 150 150 T T T 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 93 93 93 153 153 153 F F F 28 560-579 94 94 94 155 155 155 F F F 29 580-599 94 94 94 155 155 155 F F F Max gen: 94, Max flush: 156 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 94 94 94 155 155 155 F F F 2 040-059 93 93 93 153 153 153 F F F 3 060-079 94 94 94 155 155 155 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 92 92 92 151 151 151 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 93 93 93 153 153 153 F F F 8 160-179 94 94 94 155 155 155 F F F 9 180-199 92 92 92 152 152 152 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 94 94 94 155 155 155 F F F 13 260-279 94 94 94 154 154 154 F F F 14 280-299 93 93 93 153 153 153 F F F 15 300-319 95 95 95 157 157 157 F F F 16 320-339 95 95 95 157 157 157 F F F 17 340-359 95 95 95 157 157 157 F F F 18 360-379 93 93 93 153 153 153 F F F 19 380-399 88 88 88 144 144 144 F F F 20 400-419 92 92 92 151 151 151 F F F 21 420-439 95 95 95 157 157 157 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 89 89 89 146 146 146 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 93 93 93 153 153 153 F F F 28 560-579 94 94 94 155 155 155 F F F 29 580-599 94 94 94 155 155 155 F F F Max gen: 95, Max flush: 157 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:20.932496853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:20.974297548Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:20.974715847Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.005486581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006150248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006191899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006208035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006225472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} {"msg":"Crucible e895c03d-d5eb-4f87-a028-a341554b41f8 has session id: 5b927562-6638-43d6-8764-78306979de38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006718834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Upstairs opts: Upstairs UUID: e895c03d-d5eb-4f87-a028-a341554b41f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.006766804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Crucible stats registered with UUID: e895c03d-d5eb-4f87-a028-a341554b41f8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.00678625Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:21.006800107Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007209848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007242951Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} {"msg":"e895c03d-d5eb-4f87-a028-a341554b41f8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.00729574Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007327423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007352367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007367184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007381431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007394828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007420303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007503675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007530629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007556883Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"ds_connection connected from Ok(127.0.0.1:60949)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007583637Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"ds_connection connected from Ok(127.0.0.1:53223)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007607162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"ds_connection connected from Ok(127.0.0.1:35234)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.007649433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008241205Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008280616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008451139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008918178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008947652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.008971057Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.00951031Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009612238Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[0]R flush_numbers[0..12]: [141, 155, 153, 155, 139, 151, 157, 153, 155, 152, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009713956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[0]R generation[0..12]: [86, 94, 93, 94, 83, 92, 95, 93, 94, 92, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009734032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009748698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[1]R flush_numbers[0..12]: [141, 155, 153, 155, 139, 151, 157, 153, 155, 152, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009762715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[1]R generation[0..12]: [86, 94, 93, 94, 83, 92, 95, 93, 94, 92, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009776022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009790569Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[2]R flush_numbers[0..12]: [141, 155, 153, 155, 139, 151, 157, 153, 155, 152, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009804126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[2]R generation[0..12]: [86, 94, 93, 94, 83, 92, 95, 93, 94, 92, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009817683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009835629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009849307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009867692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.00988044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009893827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"":"downstairs","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009906794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009918441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"0","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009931329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"1","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009944696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"client":"2","":"downstairs","session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009957333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009969301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"e895c03d-d5eb-4f87-a028-a341554b41f8 is now active with session: 5b927562-6638-43d6-8764-78306979de38","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009981648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.009995685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568,"session_id":"5b927562-6638-43d6-8764-78306979de38"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:21.01001776Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 343 len 12288 data: 9 9 10 02/30 Write block 482 len 24576 data: 8 9 8 7 5 4 03/30 Flush 04/30 Write block 253 len 8192 data: 5 5 05/30 Read block 431 len 40960 06/30 Write block 319 len 20480 data: 8 8 9 6 7 07/30 Write block 386 len 40960 data: 8 9 8 7 7 7 7 5 6 6 08/30 Read block 276 len 40960 09/30 Read block 559 len 4096 10/30 Read block 155 len 4096 11/30 Read block 350 len 12288 12/30 Write block 402 len 16384 data: 7 7 6 5 13/30 Read block 422 len 12288 14/30 Write block 44 len 32768 data: 5 7 7 6 7 7 8 7 15/30 Read block 410 len 12288 16/30 Write block 111 len 8192 data: 8 8 17/30 Flush 18/30 Write block 183 len 12288 data: 9 10 10 19/30 Read block 490 len 24576 20/30 Read block 236 len 24576 21/30 Read block 195 len 24576 22/30 Read block 408 len 40960 23/30 Write block 273 len 8192 data: 9 9 24/30 Write block 155 len 4096 data: 6 25/30 Read block 234 len 4096 26/30 Write block 429 len 28672 data: 6 5 5 5 4 3 4 27/30 Write block 595 len 4096 data: 4 28/30 Read block 218 len 20480 29/30 Write block 558 len 36864 data: 9 10 9 8 8 6 7 10 11 30/30 Read block 83 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:21.029669033Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1568} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 94 94 94 155 155 155 F F F 2 040-059 96 96 96 159 159 159 F F F 3 060-079 94 94 94 155 155 155 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 93 93 93 153 153 153 T T T 8 160-179 94 94 94 155 155 155 F F F 9 180-199 92 92 92 152 152 152 T T T 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 96 96 96 159 159 159 F F F 13 260-279 94 94 94 154 154 154 T T T 14 280-299 93 93 93 153 153 153 F F F 15 300-319 96 96 96 159 159 159 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 96 96 96 158 158 158 F F F 18 360-379 93 93 93 153 153 153 F F F 19 380-399 96 96 96 159 159 159 F F F 20 400-419 96 96 96 159 159 159 F F F 21 420-439 95 95 95 157 157 157 T T T 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 93 93 93 153 153 153 T T T 28 560-579 94 94 94 155 155 155 T T T 29 580-599 94 94 94 155 155 155 T T T Max gen: 96, Max flush: 159 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 94 94 94 155 155 155 F F F 2 040-059 96 96 96 159 159 159 F F F 3 060-079 94 94 94 155 155 155 F F F 4 080-099 83 83 83 139 139 139 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 97 97 97 160 160 160 F F F 8 160-179 94 94 94 155 155 155 F F F 9 180-199 97 97 97 160 160 160 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 96 96 96 159 159 159 F F F 13 260-279 97 97 97 160 160 160 F F F 14 280-299 93 93 93 153 153 153 F F F 15 300-319 96 96 96 159 159 159 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 96 96 96 158 158 158 F F F 18 360-379 93 93 93 153 153 153 F F F 19 380-399 96 96 96 159 159 159 F F F 20 400-419 96 96 96 159 159 159 F F F 21 420-439 97 97 97 160 160 160 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 97 97 97 160 160 160 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 97, Max flush: 160 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.333926036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.379554142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.380259109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.38030022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.380315337Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.380330014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} {"msg":"Crucible 251792f4-ab4d-4f84-be28-393460668518 has session id: 1545c754-ecb9-4921-9366-1813e3b9340f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.380874865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Upstairs opts: Upstairs UUID: 251792f4-ab4d-4f84-be28-393460668518, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.380923605Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Crucible stats registered with UUID: 251792f4-ab4d-4f84-be28-393460668518","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.38094768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:25.380967865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381435114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381461868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} {"msg":"251792f4-ab4d-4f84-be28-393460668518 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381527534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.38154496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381567705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381581323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381595009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381608346Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381633071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381673552Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381699017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.38173083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47216)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381766072Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38988)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381792687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54904)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.381818021Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.382453214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.382481277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.383143214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.383175877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.383844302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.383870796Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.383885583Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384427586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[0]R flush_numbers[0..12]: [141, 155, 159, 155, 139, 159, 157, 160, 155, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384912061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[0]R generation[0..12]: [86, 94, 96, 94, 83, 96, 95, 97, 94, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384932217Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384950043Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[1]R flush_numbers[0..12]: [141, 155, 159, 155, 139, 159, 157, 160, 155, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.38496441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[1]R generation[0..12]: [86, 94, 96, 94, 83, 96, 95, 97, 94, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384978167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.384991734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[2]R flush_numbers[0..12]: [141, 155, 159, 155, 139, 159, 157, 160, 155, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385005071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[2]R generation[0..12]: [86, 94, 96, 94, 83, 96, 95, 97, 94, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385018158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385031455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385043902Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385067757Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385091742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385117206Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"":"downstairs","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385156378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385180493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"0","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385207127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"1","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385233571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"client":"2","":"downstairs","session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385263795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385288769Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"251792f4-ab4d-4f84-be28-393460668518 is now active with session: 1545c754-ecb9-4921-9366-1813e3b9340f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385318453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385342308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576,"session_id":"1545c754-ecb9-4921-9366-1813e3b9340f"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:25.385355855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 36864 data: 9 9 9 10 7 7 8 7 6 02/30 Read block 186 len 4096 03/30 Read block 509 len 20480 04/30 Read block 60 len 20480 05/30 Read block 592 len 4096 06/30 Write block 273 len 40960 data: 10 10 9 9 9 10 9 11 13 12 07/30 Write block 89 len 28672 data: 9 7 7 5 5 4 4 08/30 Read block 526 len 8192 09/30 Read block 405 len 20480 10/30 Flush 11/30 Write block 272 len 8192 data: 9 11 12/30 Read block 185 len 36864 13/30 Read block 368 len 8192 14/30 Write block 397 len 32768 data: 6 7 8 8 9 8 8 7 15/30 Write block 32 len 20480 data: 10 8 12 9 9 16/30 Write block 161 len 16384 data: 11 11 8 8 17/30 Read block 302 len 8192 18/30 Flush 19/30 Write block 368 len 32768 data: 6 5 8 8 8 8 6 8 20/30 Write block 295 len 28672 data: 8 8 7 5 4 6 8 21/30 Write block 41 len 16384 data: 7 5 4 6 22/30 Read block 589 len 32768 23/30 Read block 116 len 8192 24/30 Write block 64 len 28672 data: 8 8 10 9 7 9 7 25/30 Read block 66 len 28672 26/30 Read block 510 len 24576 27/30 Read block 479 len 28672 28/30 Read block 181 len 20480 29/30 Read block 254 len 12288 30/30 Write block 270 len 24576 data: 7 8 10 12 11 10 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:25.405655108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1576} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 98 98 98 162 162 162 F F F 2 040-059 96 96 96 159 159 159 T T T 3 060-079 94 94 94 155 155 155 T T T 4 080-099 98 98 98 161 161 161 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 97 97 97 160 160 160 F F F 8 160-179 98 98 98 162 162 162 F F F 9 180-199 97 97 97 160 160 160 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 96 96 96 159 159 159 F F F 13 260-279 98 98 98 162 162 162 T T T 14 280-299 98 98 98 161 161 161 T T T 15 300-319 98 98 98 161 161 161 T T T 16 320-339 96 96 96 159 159 159 F F F 17 340-359 96 96 96 158 158 158 F F F 18 360-379 93 93 93 153 153 153 T T T 19 380-399 98 98 98 162 162 162 F F F 20 400-419 98 98 98 162 162 162 F F F 21 420-439 97 97 97 160 160 160 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 97 97 97 160 160 160 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 98, Max flush: 162 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 98 98 98 162 162 162 F F F 2 040-059 99 99 99 163 163 163 F F F 3 060-079 99 99 99 163 163 163 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 97 97 97 160 160 160 F F F 8 160-179 98 98 98 162 162 162 F F F 9 180-199 97 97 97 160 160 160 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 96 96 96 159 159 159 F F F 13 260-279 99 99 99 163 163 163 F F F 14 280-299 99 99 99 163 163 163 F F F 15 300-319 99 99 99 163 163 163 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 96 96 96 158 158 158 F F F 18 360-379 99 99 99 163 163 163 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 98 98 98 162 162 162 F F F 21 420-439 97 97 97 160 160 160 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 F F F 27 540-559 97 97 97 160 160 160 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 99, Max flush: 163 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.650506018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.694941472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.695821481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.695864132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.695879179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.695893416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} {"msg":"Crucible a10ca08d-b2fc-456e-99a8-6acafbf3ae33 has session id: 798ba9e5-82cd-426e-be00-481e4a5bc02b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.696439878Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Upstairs opts: Upstairs UUID: a10ca08d-b2fc-456e-99a8-6acafbf3ae33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.696476319Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Crucible stats registered with UUID: a10ca08d-b2fc-456e-99a8-6acafbf3ae33","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.696502284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:29.696525309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.696939139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.696962904Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} {"msg":"a10ca08d-b2fc-456e-99a8-6acafbf3ae33 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697025301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697051565Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697097455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697112362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697125759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697139406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697152813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697172559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697198143Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58884)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697221718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62800)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697262649Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37001)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.69730562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697893093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.697933104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.698567486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.69859796Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.699260147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.69929068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.699507933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.699994368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[0]R flush_numbers[0..12]: [141, 162, 163, 163, 161, 159, 157, 160, 162, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700506877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[0]R generation[0..12]: [86, 98, 99, 99, 98, 96, 95, 97, 98, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700527122Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700541129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[1]R flush_numbers[0..12]: [141, 162, 163, 163, 161, 159, 157, 160, 162, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700554466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[1]R generation[0..12]: [86, 98, 99, 99, 98, 96, 95, 97, 98, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700567663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.70058108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[2]R flush_numbers[0..12]: [141, 162, 163, 163, 161, 159, 157, 160, 162, 160, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700594567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[2]R generation[0..12]: [86, 98, 99, 99, 98, 96, 95, 97, 98, 97, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700607415Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700633589Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:42:29.700647406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700723939Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700739516Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700751983Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"":"downstairs","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.70076503Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700776998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"0","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700789745Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"1","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700802342Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"client":"2","":"downstairs","session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.70081526Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700826997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"a10ca08d-b2fc-456e-99a8-6acafbf3ae33 is now active with session: 798ba9e5-82cd-426e-be00-481e4a5bc02b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700839564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700851182Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584,"session_id":"798ba9e5-82cd-426e-be00-481e4a5bc02b"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:29.700863009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 24576 data: 6 6 5 5 5 6 02/30 Write block 480 len 8192 data: 11 11 03/30 Write block 341 len 36864 data: 8 10 10 10 11 10 8 7 8 04/30 Write block 489 len 24576 data: 7 6 6 6 4 7 05/30 Read block 1 len 28672 06/30 Read block 69 len 8192 07/30 Write block 109 len 32768 data: 7 8 9 9 6 5 6 7 08/30 Write block 474 len 28672 data: 6 6 7 8 8 8 12 09/30 Write block 256 len 36864 data: 8 10 10 9 8 7 7 7 10 10/30 Write block 309 len 4096 data: 6 11/30 Write block 188 len 12288 data: 11 11 12 12/30 Read block 442 len 40960 13/30 Write block 363 len 20480 data: 7 6 6 6 6 14/30 Write block 406 len 28672 data: 5 7 8 10 9 10 10 15/30 Write block 21 len 8192 data: 7 7 16/30 Read block 315 len 12288 17/30 Read block 243 len 36864 18/30 Flush 19/30 Write block 37 len 20480 data: 8 9 9 8 8 20/30 Flush 21/30 Flush 22/30 Write block 521 len 24576 data: 8 8 8 7 9 9 23/30 Write block 258 len 32768 data: 11 10 9 8 8 8 11 11 24/30 Write block 269 len 12288 data: 10 8 9 25/30 Read block 9 len 12288 26/30 Read block 113 len 16384 27/30 Write block 146 len 8192 data: 10 10 28/30 Read block 106 len 24576 29/30 Write block 318 len 4096 data: 7 30/30 Write block 359 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:29.718156083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1584} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 99 99 99 163 163 163 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 97 97 97 160 160 160 T T T 8 160-179 98 98 98 162 162 162 F F F 9 180-199 100 100 100 164 164 164 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 100 100 100 164 164 164 T T T 13 260-279 100 100 100 164 164 164 T T T 14 280-299 99 99 99 163 163 163 F F F 15 300-319 100 100 100 164 164 164 T T T 16 320-339 96 96 96 159 159 159 F F F 17 340-359 100 100 100 164 164 164 T T T 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 97 97 97 160 160 160 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 100 100 100 164 164 164 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 92 92 92 151 151 151 T T T 27 540-559 97 97 97 160 160 160 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 100, Max flush: 165 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 99 99 99 163 163 163 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 95 95 95 157 157 157 F F F 7 140-159 101 101 101 166 166 166 F F F 8 160-179 98 98 98 162 162 162 F F F 9 180-199 100 100 100 164 164 164 F F F 10 200-219 93 93 93 153 153 153 F F F 11 220-239 94 94 94 155 155 155 F F F 12 240-259 101 101 101 166 166 166 F F F 13 260-279 101 101 101 166 166 166 F F F 14 280-299 99 99 99 163 163 163 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 101 101 101 166 166 166 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 97 97 97 160 160 160 F F F 22 440-459 94 94 94 156 156 156 F F F 23 460-479 100 100 100 164 164 164 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 92 92 92 151 151 151 F F F 26 520-539 101 101 101 166 166 166 F F F 27 540-559 97 97 97 160 160 160 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 101, Max flush: 166 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:43.96350045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:44.005152027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.005711296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.036646815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.037298424Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.037343134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.03736218Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.037380316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} {"msg":"Crucible 6fdf7e86-0598-4e27-b58f-3cb2d2862bf1 has session id: 9fb9e073-42eb-4aa8-ab5e-899307ec8aa3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.037918089Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Upstairs opts: Upstairs UUID: 6fdf7e86-0598-4e27-b58f-3cb2d2862bf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.037975117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Crucible stats registered with UUID: 6fdf7e86-0598-4e27-b58f-3cb2d2862bf1","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038004461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:44.038031035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038408543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038435707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} {"msg":"6fdf7e86-0598-4e27-b58f-3cb2d2862bf1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038518809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038537025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.03856016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038574487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038588764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.03860991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038636154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038660379Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038687073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038713087Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56973)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038734912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56954)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038752159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43861)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.038971421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.03943942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.039480271Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.039563493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.04012948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040154815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.04017681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040733359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040769262Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[0]R flush_numbers[0..12]: [141, 165, 165, 163, 161, 164, 157, 166, 162, 164, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040785408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[0]R generation[0..12]: [86, 100, 100, 99, 98, 100, 95, 101, 98, 100, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040800085Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040814132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[1]R flush_numbers[0..12]: [141, 165, 165, 163, 161, 164, 157, 166, 162, 164, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040831868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[1]R generation[0..12]: [86, 100, 100, 99, 98, 100, 95, 101, 98, 100, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040846105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040859932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[2]R flush_numbers[0..12]: [141, 165, 165, 163, 161, 164, 157, 166, 162, 164, 153, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040873199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[2]R generation[0..12]: [86, 100, 100, 99, 98, 100, 95, 101, 98, 100, 93, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040886556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040899914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040912221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040924838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040937315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040955242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"":"downstairs","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040967879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040979696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"0","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.040993113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"1","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.041006071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"client":"2","":"downstairs","session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.041018798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.041032705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"6fdf7e86-0598-4e27-b58f-3cb2d2862bf1 is now active with session: 9fb9e073-42eb-4aa8-ab5e-899307ec8aa3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.041045472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.04105708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592,"session_id":"9fb9e073-42eb-4aa8-ab5e-899307ec8aa3"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:44.041068877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 24576 data: 10 10 8 4 5 6 02/30 Read block 553 len 12288 03/30 Write block 431 len 28672 data: 6 6 5 4 5 6 6 04/30 Read block 215 len 36864 05/30 Write block 430 len 24576 data: 6 7 7 6 5 6 06/30 Read block 171 len 8192 07/30 Read block 557 len 16384 08/30 Read block 77 len 4096 09/30 Read block 208 len 32768 10/30 Read block 301 len 36864 11/30 Write block 501 len 40960 data: 7 9 9 10 11 9 7 7 7 7 12/30 Read block 533 len 24576 13/30 Flush 14/30 Write block 525 len 28672 data: 10 10 6 8 11 11 12 15/30 Write block 208 len 20480 data: 11 7 8 9 8 16/30 Read block 514 len 40960 17/30 Write block 237 len 24576 data: 8 11 11 10 9 11 18/30 Write block 244 len 32768 data: 7 6 5 4 3 3 4 5 19/30 Read block 286 len 36864 20/30 Flush 21/30 Write block 442 len 32768 data: 5 5 6 9 9 7 9 8 22/30 Read block 131 len 32768 23/30 Write block 554 len 8192 data: 8 8 24/30 Write block 262 len 8192 data: 9 9 25/30 Write block 344 len 16384 data: 11 12 11 9 26/30 Read block 573 len 12288 27/30 Read block 55 len 8192 28/30 Write block 68 len 24576 data: 8 10 8 7 8 9 29/30 Read block 212 len 28672 30/30 Read block 559 len 40960 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:44.061720473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1592} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 99 99 99 163 163 163 T T T 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 101 101 101 166 166 166 F F F 8 160-179 98 98 98 162 162 162 F F F 9 180-199 100 100 100 164 164 164 F F F 10 200-219 102 102 102 168 168 168 F F F 11 220-239 102 102 102 168 168 168 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 101 101 101 166 166 166 T T T 14 280-299 99 99 99 163 163 163 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 101 101 101 166 166 166 T T T 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 102 102 102 167 167 167 F F F 22 440-459 94 94 94 156 156 156 T T T 23 460-479 100 100 100 164 164 164 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 102 102 102 167 167 167 F F F 26 520-539 102 102 102 168 168 168 F F F 27 540-559 97 97 97 160 160 160 T T T 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 102, Max flush: 168 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 103 103 103 169 169 169 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 101 101 101 166 166 166 F F F 8 160-179 98 98 98 162 162 162 F F F 9 180-199 100 100 100 164 164 164 F F F 10 200-219 102 102 102 168 168 168 F F F 11 220-239 102 102 102 168 168 168 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 103 103 103 169 169 169 F F F 14 280-299 99 99 99 163 163 163 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 96 96 96 159 159 159 F F F 17 340-359 103 103 103 169 169 169 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 102 102 102 167 167 167 F F F 22 440-459 103 103 103 169 169 169 F F F 23 460-479 100 100 100 164 164 164 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 102 102 102 167 167 167 F F F 26 520-539 102 102 102 168 168 168 F F F 27 540-559 103 103 103 169 169 169 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 103, Max flush: 169 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.333677882Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.37837521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.379186654Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.379236163Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.379252919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.379267416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} {"msg":"Crucible 1ce0e3d9-4e53-4ca9-8383-bbac30720c83 has session id: f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.37984852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Upstairs opts: Upstairs UUID: 1ce0e3d9-4e53-4ca9-8383-bbac30720c83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.37989655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Crucible stats registered with UUID: 1ce0e3d9-4e53-4ca9-8383-bbac30720c83","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.379922055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:48.379938711Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380343073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380366738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} {"msg":"1ce0e3d9-4e53-4ca9-8383-bbac30720c83 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380436203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380453389Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380475624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380489121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380503258Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380520005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380534562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380547539Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380561096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380574693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:40463)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380655385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:37171)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380692268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:37604)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.380719152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.381469939Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.381499083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.381514699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382028528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382069209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382165498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382586057Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382695794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[0]R flush_numbers[0..12]: [141, 165, 165, 169, 161, 164, 167, 166, 162, 164, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382813848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[0]R generation[0..12]: [86, 100, 100, 103, 98, 100, 102, 101, 98, 100, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382834504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.38284912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[1]R flush_numbers[0..12]: [141, 165, 165, 169, 161, 164, 167, 166, 162, 164, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382863047Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[1]R generation[0..12]: [86, 100, 100, 103, 98, 100, 102, 101, 98, 100, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382876724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382890571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[2]R flush_numbers[0..12]: [141, 165, 165, 169, 161, 164, 167, 166, 162, 164, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382904138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[2]R generation[0..12]: [86, 100, 100, 103, 98, 100, 102, 101, 98, 100, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382917335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382931272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.38294434Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":Wait for a query_work_queue command to finish before sending IO "Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.382965885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.38299049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383019173Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"":"downstairs","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383045078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383067063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"0","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383085859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"1","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383103405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"client":"2","":"downstairs","session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.38312652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383153004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"1ce0e3d9-4e53-4ca9-8383-bbac30720c83 is now active with session: f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.38317345Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383186137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600,"session_id":"f85d28d7-e1a4-4ed9-87f9-1b8ccf5d40ca"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:48.383198305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 226 len 32768 data: 7 6 7 7 6 8 8 8 02/30 Read block 243 len 40960 03/30 Read block 218 len 36864 04/30 Write block 542 len 4096 data: 16 05/30 Write block 343 len 20480 data: 11 12 13 12 10 06/30 Write block 320 len 28672 data: 9 10 7 8 9 8 6 07/30 Read block 363 len 24576 08/30 Read block 551 len 4096 09/30 Read block 196 len 36864 10/30 Read block 477 len 12288 11/30 Read block 497 len 40960 12/30 Write block 5 len 24576 data: 9 10 9 10 8 10 13/30 Read block 490 len 12288 14/30 Write block 474 len 12288 data: 7 7 8 15/30 Read block 196 len 16384 16/30 Read block 373 len 20480 17/30 Read block 487 len 16384 18/30 Read block 582 len 32768 19/30 Read block 121 len 36864 20/30 Write block 423 len 8192 data: 7 9 21/30 Write block 589 len 24576 data: 4 4 5 6 6 4 22/30 Write block 517 len 24576 data: 6 3 5 7 9 9 23/30 Write block 145 len 12288 data: 9 11 11 24/30 Write block 273 len 40960 data: 13 12 11 10 10 11 10 12 14 13 25/30 Read block 45 len 24576 26/30 Write block 182 len 32768 data: 10 10 11 11 12 12 12 12 27/30 Write block 169 len 8192 data: 8 7 28/30 Write block 249 len 8192 data: 4 5 29/30 Write block 225 len 20480 data: 5 8 7 8 8 30/30 Read block 348 len 32768 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 30 30 28 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:48.402605051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1600} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 141 141 141 T T T 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 103 103 103 169 169 169 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 101 101 101 166 166 166 T T T 8 160-179 98 98 98 162 162 162 T T T 9 180-199 100 100 100 164 164 164 T T T 10 200-219 102 102 102 168 168 168 F F F 11 220-239 102 102 102 168 168 168 T T T 12 240-259 102 102 102 168 168 168 T T T 13 260-279 103 103 103 169 169 169 T T T 14 280-299 99 99 99 163 163 163 T T T 15 300-319 101 101 101 166 166 166 F F F 16 320-339 96 96 96 159 159 159 T T T 17 340-359 103 103 103 169 169 169 T T T 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 102 102 102 167 167 167 T T T 22 440-459 103 103 103 169 169 169 F F F 23 460-479 100 100 100 164 164 164 T T T 24 480-499 100 100 100 164 164 164 F F F 25 500-519 102 102 102 167 167 167 T T T 26 520-539 102 102 102 168 168 168 T T T 27 540-559 103 103 103 169 169 169 T T T 28 560-579 97 97 97 160 160 160 F F F 29 580-599 97 97 97 160 160 160 T T T Max gen: 103, Max flush: 169 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 103 103 103 169 169 169 F F F 4 080-099 98 98 98 161 161 161 F F F 5 100-119 100 100 100 164 164 164 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 104 104 104 170 170 170 F F F 9 180-199 104 104 104 170 170 170 F F F 10 200-219 102 102 102 168 168 168 F F F 11 220-239 104 104 104 170 170 170 F F F 12 240-259 104 104 104 170 170 170 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 104 104 104 170 170 170 F F F 15 300-319 101 101 101 166 166 166 F F F 16 320-339 104 104 104 170 170 170 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 98 98 98 162 162 162 F F F 20 400-419 100 100 100 164 164 164 F F F 21 420-439 104 104 104 170 170 170 F F F 22 440-459 103 103 103 169 169 169 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 104 104 104 170 170 170 F F F 26 520-539 104 104 104 170 170 170 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 97 97 97 160 160 160 F F F 29 580-599 104 104 104 170 170 170 F F F Max gen: 104, Max flush: 170 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.701350927Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.744904162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.745671106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.745712837Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.745733813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.745753818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} {"msg":"Crucible 7c1b5a1d-48e7-4c14-b281-ce5e795a9e8d has session id: b04a7b0b-901d-4e91-a17a-911948cd793d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746282604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Upstairs opts: Upstairs UUID: 7c1b5a1d-48e7-4c14-b281-ce5e795a9e8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746331523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Crucible stats registered with UUID: 7c1b5a1d-48e7-4c14-b281-ce5e795a9e8d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746354728Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:52.746376523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746865268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746893322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} {"msg":"7c1b5a1d-48e7-4c14-b281-ce5e795a9e8d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746957248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746975144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.746999488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747014136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.74703886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747065044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747094798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747120872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747147007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747364589Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41176)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747403581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36902)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747430685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33248)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.74745211Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.747973897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.748013029Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.748043153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.748612739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.748641693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.748657679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.74920851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749242373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[0]R flush_numbers[0..12]: [170, 165, 165, 169, 161, 164, 167, 170, 170, 170, 168, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749266157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[0]R generation[0..12]: [104, 100, 100, 103, 98, 100, 102, 104, 104, 104, 102, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749282604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749297311Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[1]R flush_numbers[0..12]: [170, 165, 165, 169, 161, 164, 167, 170, 170, 170, 168, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749314407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[1]R generation[0..12]: [104, 100, 100, 103, 98, 100, 102, 104, 104, 104, 102, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749328664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749342441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[2]R flush_numbers[0..12]: [170, 165, 165, 169, 161, 164, 167, 170, 170, 170, 168, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749356368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[2]R generation[0..12]: [104, 100, 100, 103, 98, 100, 102, 104, 104, 104, 102, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749369845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.74939164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749414645Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.74943907Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749457466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749479791Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"":"downstairs","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749501937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749520553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"0","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749547727Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"1","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749568812Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"client":"2","":"downstairs","session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749585099Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749597576Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"7c1b5a1d-48e7-4c14-b281-ce5e795a9e8d is now active with session: b04a7b0b-901d-4e91-a17a-911948cd793d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749610333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749622011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608,"session_id":"b04a7b0b-901d-4e91-a17a-911948cd793d"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:52.749633938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 40960 data: 8 9 9 7 9 9 9 8 10 7 02/30 Read block 111 len 32768 03/30 Write block 93 len 24576 data: 6 5 5 2 4 6 04/30 Write block 398 len 24576 data: 8 9 9 10 9 9 05/30 Flush 06/30 Write block 103 len 24576 data: 8 6 8 8 8 8 07/30 Write block 562 len 36864 data: 9 7 8 11 12 11 9 10 7 08/30 Read block 582 len 12288 09/30 Read block 491 len 36864 10/30 Write block 432 len 12288 data: 8 7 6 11/30 Write block 170 len 20480 data: 8 9 9 8 9 12/30 Write block 584 len 12288 data: 6 4 5 13/30 Flush 14/30 Write block 208 len 16384 data: 12 8 9 10 15/30 Read block 535 len 20480 16/30 Read block 143 len 32768 17/30 Read block 117 len 4096 18/30 Read block 461 len 32768 19/30 Write block 216 len 40960 data: 4 4 4 6 4 4 4 7 7 6 20/30 Read block 44 len 20480 21/30 Read block 107 len 32768 22/30 Write block 325 len 36864 data: 9 7 6 9 9 9 9 9 7 23/30 Read block 369 len 32768 24/30 Write block 86 len 20480 data: 10 10 9 10 8 25/30 Read block 316 len 20480 26/30 Read block 401 len 40960 27/30 Write block 514 len 12288 data: 5 4 5 28/30 Write block 292 len 40960 data: 8 7 9 9 9 8 6 5 7 9 29/30 Read block 160 len 16384 30/30 Write block 597 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 16 16 14 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:52.77104413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1608} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 103 103 103 169 169 169 F F F 4 080-099 106 106 106 171 171 171 T T T 5 100-119 106 106 106 172 172 172 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 104 104 104 170 170 170 F F F 10 200-219 102 102 102 168 168 168 T T T 11 220-239 106 106 106 171 171 171 T T T 12 240-259 104 104 104 170 170 170 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 104 104 104 170 170 170 T T T 15 300-319 101 101 101 166 166 166 T T T 16 320-339 104 104 104 170 170 170 T T T 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 106 106 106 171 171 171 F F F 20 400-419 106 106 106 171 171 171 F F F 21 420-439 106 106 106 172 172 172 F F F 22 440-459 103 103 103 169 169 169 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 104 104 104 170 170 170 T T T 26 520-539 104 104 104 170 170 170 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 106 106 106 172 172 172 T T T Max gen: 106, Max flush: 172 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 100 100 100 165 165 165 F F F 3 060-079 103 103 103 169 169 169 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 106 106 106 172 172 172 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 104 104 104 170 170 170 F F F 10 200-219 107 107 107 173 173 173 F F F 11 220-239 107 107 107 173 173 173 F F F 12 240-259 104 104 104 170 170 170 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 107 107 107 173 173 173 F F F 15 300-319 107 107 107 173 173 173 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 106 106 106 171 171 171 F F F 20 400-419 106 106 106 171 171 171 F F F 21 420-439 106 106 106 172 172 172 F F F 22 440-459 103 103 103 169 169 169 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 107 107 107 173 173 173 F F F 26 520-539 104 104 104 170 170 170 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 107, Max flush: 173 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.026303425Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:57.068008531Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.068408184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.099383344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.099988983Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.100036643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.10005059Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.100064986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} {"msg":"Crucible 4cd23126-78c8-456d-8024-6ca7ca4ade87 has session id: 02c15e49-bef8-47cb-a492-ec87894483d3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.100715336Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Upstairs opts: Upstairs UUID: 4cd23126-78c8-456d-8024-6ca7ca4ade87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.100750248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Crucible stats registered with UUID: 4cd23126-78c8-456d-8024-6ca7ca4ade87","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.100764805Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:57.100777752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101173696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101197301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} {"msg":"4cd23126-78c8-456d-8024-6ca7ca4ade87 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101264826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101281913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101305148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101319575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101333352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101346339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101359776Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101378732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101395258Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101408865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34739)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.10157431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49410)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101614991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51866)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.101643544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.102386483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.102534642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.102581851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103014138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103093061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103133022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103539134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103682153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[0]R flush_numbers[0..12]: [170, 165, 165, 169, 173, 172, 167, 170, 172, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103706528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[0]R generation[0..12]: [104, 100, 100, 103, 107, 106, 102, 104, 106, 104, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103722874Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103737581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[1]R flush_numbers[0..12]: [170, 165, 165, 169, 173, 172, 167, 170, 172, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103751708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[1]R generation[0..12]: [104, 100, 100, 103, 107, 106, 102, 104, 106, 104, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103765325Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103778872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[2]R flush_numbers[0..12]: [170, 165, 165, 169, 173, 172, 167, 170, 172, 170, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103792099Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[2]R generation[0..12]: [104, 100, 100, 103, 107, 106, 102, 104, 106, 104, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103805186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103822363Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.10383565Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-23T17:42:57.103854845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103867503Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.10387994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"":"downstairs","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103892627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103904135Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"0","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103917012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"1","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103934208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"client":"2","":"downstairs","session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103957553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.10397392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"4cd23126-78c8-456d-8024-6ca7ca4ade87 is now active with session: 02c15e49-bef8-47cb-a492-ec87894483d3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103986617Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.103998045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616,"session_id":"02c15e49-bef8-47cb-a492-ec87894483d3"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:42:57.104009702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 4096 data: 12 02/30 Write block 306 len 16384 data: 7 7 7 7 03/30 Write block 214 len 32768 data: 5 4 5 5 5 7 5 5 04/30 Flush 05/30 Write block 443 len 20480 data: 6 7 10 10 8 06/30 Write block 73 len 12288 data: 10 7 6 07/30 Read block 67 len 8192 08/30 Write block 531 len 32768 data: 13 11 12 11 9 7 11 9 09/30 Write block 506 len 40960 data: 10 8 8 8 8 7 6 5 6 5 10/30 Write block 502 len 12288 data: 10 10 11 11/30 Read block 395 len 28672 12/30 Write block 236 len 12288 data: 8 9 12 13/30 Read block 7 len 32768 14/30 Read block 565 len 12288 15/30 Read block 26 len 20480 16/30 Flush 17/30 Write block 394 len 24576 data: 7 7 6 7 9 10 18/30 Flush 19/30 Write block 294 len 28672 data: 10 10 10 9 7 6 8 20/30 Read block 204 len 12288 21/30 Read block 467 len 36864 22/30 Write block 265 len 24576 data: 12 10 11 12 11 9 23/30 Write block 197 len 32768 data: 7 5 7 7 8 11 11 11 24/30 Read block 23 len 12288 25/30 Read block 76 len 4096 26/30 Flush 27/30 Write block 278 len 8192 data: 12 11 28/30 Write block 200 len 40960 data: 8 9 12 12 12 12 13 13 13 9 29/30 Write block 221 len 36864 data: 6 5 8 8 7 9 9 10 10 30/30 Write block 50 len 40960 data: 9 8 7 6 7 7 7 8 9 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:42:57.122746793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1616} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 108 108 108 174 174 174 F F F 2 040-059 100 100 100 165 165 165 T T T 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 106 106 106 172 172 172 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 108 108 108 177 177 177 F F F 10 200-219 108 108 108 177 177 177 T T T 11 220-239 108 108 108 175 175 175 T T T 12 240-259 104 104 104 170 170 170 F F F 13 260-279 108 108 108 177 177 177 T T T 14 280-299 108 108 108 177 177 177 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 108 108 108 176 176 176 F F F 20 400-419 106 106 106 171 171 171 F F F 21 420-439 106 106 106 172 172 172 F F F 22 440-459 108 108 108 175 175 175 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 108 108 108 175 175 175 F F F 26 520-539 108 108 108 175 175 175 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 108, Max flush: 177 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 108 108 108 174 174 174 F F F 2 040-059 109 109 109 178 178 178 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 106 106 106 172 172 172 F F F 6 120-139 102 102 102 167 167 167 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 108 108 108 177 177 177 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 109 109 109 178 178 178 F F F 12 240-259 104 104 104 170 170 170 F F F 13 260-279 109 109 109 178 178 178 F F F 14 280-299 108 108 108 177 177 177 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 108 108 108 176 176 176 F F F 20 400-419 106 106 106 171 171 171 F F F 21 420-439 106 106 106 172 172 172 F F F 22 440-459 108 108 108 175 175 175 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 108 108 108 175 175 175 F F F 26 520-539 108 108 108 175 175 175 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 109, Max flush: 178 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.39171008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:01.434172102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.434744178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.466407079Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.467337487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.46737346Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.467388407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.467403313Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} {"msg":"Crucible 33cbf321-6a4d-408d-9edb-70fd8f57313b has session id: a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468040005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Upstairs opts: Upstairs UUID: 33cbf321-6a4d-408d-9edb-70fd8f57313b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468085856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Crucible stats registered with UUID: 33cbf321-6a4d-408d-9edb-70fd8f57313b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.46811046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:01.468125197Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468490788Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468520941Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} {"msg":"33cbf321-6a4d-408d-9edb-70fd8f57313b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468568051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468593965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.46861784Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468632417Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468652853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468677517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468705121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468747932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468779735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468799031Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"ds_connection connected from Ok(127.0.0.1:45111)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.468835433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"ds_connection connected from Ok(127.0.0.1:39690)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.469014644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"ds_connection connected from Ok(127.0.0.1:35680)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.469054726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.469598798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.469637339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.469666733Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.470301616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.470329829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.470345606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.47088222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.470987477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[0]R flush_numbers[0..12]: [170, 174, 178, 175, 173, 172, 167, 170, 172, 177, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471030558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[0]R generation[0..12]: [104, 108, 109, 108, 107, 106, 102, 104, 106, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471047874Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.47106636Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[1]R flush_numbers[0..12]: [170, 174, 178, 175, 173, 172, 167, 170, 172, 177, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471080897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[1]R generation[0..12]: [104, 108, 109, 108, 107, 106, 102, 104, 106, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471094774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471108581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[2]R flush_numbers[0..12]: [170, 174, 178, 175, 173, 172, 167, 170, 172, 177, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471122118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[2]R generation[0..12]: [104, 108, 109, 108, 107, 106, 102, 104, 106, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471135825Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471149692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:01.471162339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471179225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471192012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.47120457Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"":"downstairs","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471217327Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471228844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"0","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471241692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"1","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471256558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"client":"2","":"downstairs","session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471269486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471281303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"33cbf321-6a4d-408d-9edb-70fd8f57313b is now active with session: a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471293681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471305358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624,"session_id":"a1c0e7b7-f6c2-4e97-b3c0-88aad73cb849"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:01.471317065Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 29 len 32768 data: 9 9 10 11 9 13 10 10 02/30 Read block 66 len 40960 03/30 Flush 04/30 Flush 05/30 Write block 280 len 32768 data: 13 15 14 12 10 9 10 10 06/30 Flush 07/30 Write block 115 len 4096 data: 7 08/30 Write block 294 len 12288 data: 11 11 11 09/30 Read block 523 len 28672 10/30 Flush 11/30 Write block 427 len 8192 data: 10 7 12/30 Write block 106 len 24576 data: 9 9 9 8 9 10 13/30 Write block 515 len 16384 data: 6 6 7 4 14/30 Read block 519 len 40960 15/30 Write block 296 len 12288 data: 12 10 8 16/30 Read block 217 len 40960 17/30 Write block 111 len 40960 data: 11 10 7 6 8 8 6 8 10 11 18/30 Read block 569 len 24576 19/30 Write block 465 len 16384 data: 6 8 10 10 20/30 Write block 266 len 36864 data: 11 12 13 12 10 10 11 14 13 21/30 Write block 398 len 36864 data: 10 11 10 11 10 10 8 6 6 22/30 Read block 575 len 28672 23/30 Write block 523 len 28672 data: 9 8 11 11 7 9 12 24/30 Flush 25/30 Write block 444 len 16384 data: 8 11 11 9 26/30 Write block 183 len 20480 data: 11 12 12 13 13 27/30 Write block 125 len 32768 data: 9 5 6 7 6 5 5 6 28/30 Read block 240 len 28672 29/30 Write block 418 len 4096 data: 10 30/30 Write block 384 len 24576 data: 10 8 9 10 9 8 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:01.491717726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1624} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 110 110 110 179 179 179 F F F 2 040-059 109 109 109 178 178 178 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 110 110 110 183 183 183 F F F 6 120-139 110 110 110 183 183 183 T T T 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 108 108 108 177 177 177 T T T 10 200-219 109 109 109 178 178 178 F F F 11 220-239 109 109 109 178 178 178 F F F 12 240-259 104 104 104 170 170 170 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 110 110 110 183 183 183 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 110 110 110 183 183 183 T T T 20 400-419 110 110 110 183 183 183 T T T 21 420-439 110 110 110 183 183 183 F F F 22 440-459 108 108 108 175 175 175 T T T 23 460-479 110 110 110 183 183 183 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 110 110 110 183 183 183 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 110, Max flush: 183 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 170 170 170 F F F 1 020-039 110 110 110 179 179 179 F F F 2 040-059 109 109 109 178 178 178 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 110 110 110 183 183 183 F F F 6 120-139 111 111 111 184 184 184 F F F 7 140-159 104 104 104 170 170 170 F F F 8 160-179 106 106 106 172 172 172 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 109 109 109 178 178 178 F F F 12 240-259 104 104 104 170 170 170 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 110 110 110 183 183 183 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 111 111 111 184 184 184 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 110 110 110 183 183 183 F F F 22 440-459 111 111 111 184 184 184 F F F 23 460-479 110 110 110 183 183 183 F F F 24 480-499 100 100 100 164 164 164 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 110 110 110 183 183 183 F F F 27 540-559 104 104 104 170 170 170 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 111, Max flush: 184 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.774657095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.818601566Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.819574865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.819621175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.819637232Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.819652698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} {"msg":"Crucible 73c52881-7504-48ef-9944-7dc59f359387 has session id: 16fca8f9-2198-4a82-a52d-520717d9988f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820302258Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Upstairs opts: Upstairs UUID: 73c52881-7504-48ef-9944-7dc59f359387, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82033947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Crucible stats registered with UUID: 73c52881-7504-48ef-9944-7dc59f359387","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820355596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:05.820368923Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820816586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82084237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} {"msg":"73c52881-7504-48ef-9944-7dc59f359387 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820905877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820923413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820949667Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820969973Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82098493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.820997937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821011824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821024681Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821038498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821068492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57519)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821148064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63762)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821204172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60634)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821244983Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821865699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.821908739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.822532784Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.822576015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.823193721Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.823298929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82398432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.824458267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[0]R flush_numbers[0..12]: [170, 179, 178, 175, 173, 183, 184, 170, 172, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.824937783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[0]R generation[0..12]: [104, 110, 109, 108, 107, 110, 111, 104, 106, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.824956969Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.824971476Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[1]R flush_numbers[0..12]: [170, 179, 178, 175, 173, 183, 184, 170, 172, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.824984993Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[1]R generation[0..12]: [104, 110, 109, 108, 107, 110, 111, 104, 106, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825011187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825025714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[2]R flush_numbers[0..12]: [170, 179, 178, 175, 173, 183, 184, 170, 172, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825039172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[2]R generation[0..12]: [104, 110, 109, 108, 107, 110, 111, 104, 106, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825052729Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825066566Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825079243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825096509Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:05.825109886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82513622Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"":"downstairs","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825149188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825161065Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"0","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825174012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"1","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.82518692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"client":"2","":"downstairs","session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825199677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825211734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"73c52881-7504-48ef-9944-7dc59f359387 is now active with session: 16fca8f9-2198-4a82-a52d-520717d9988f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825224201Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825235849Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632,"session_id":"16fca8f9-2198-4a82-a52d-520717d9988f"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:05.825248226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 14 len 40960 data: 9 9 10 8 8 7 8 8 8 9 02/30 Read block 17 len 40960 03/30 Flush 04/30 Write block 242 len 28672 data: 12 9 8 7 6 5 4 05/30 Flush 06/30 Write block 424 len 28672 data: 10 8 9 11 8 7 7 07/30 Write block 154 len 12288 data: 6 7 10 08/30 Read block 400 len 4096 09/30 Write block 538 len 16384 data: 10 14 15 17 10/30 Read block 472 len 16384 11/30 Read block 203 len 24576 12/30 Read block 575 len 32768 13/30 Flush 14/30 Write block 35 len 32768 data: 11 11 9 10 10 9 9 6 15/30 Write block 288 len 36864 data: 9 8 7 10 9 8 12 12 13 16/30 Read block 26 len 40960 17/30 Flush 18/30 Write block 531 len 40960 data: 14 12 13 12 10 8 12 11 15 16 19/30 Write block 123 len 12288 data: 11 11 10 20/30 Write block 483 len 12288 data: 10 9 8 21/30 Write block 10 len 28672 data: 11 11 12 12 10 10 11 22/30 Write block 285 len 40960 data: 10 11 11 10 9 8 11 10 9 13 23/30 Read block 570 len 32768 24/30 Write block 486 len 28672 data: 6 5 6 8 7 7 7 25/30 Read block 489 len 12288 26/30 Write block 493 len 24576 data: 5 8 6 8 8 7 27/30 Read block 560 len 24576 28/30 Read block 0 len 40960 29/30 Write block 177 len 12288 data: 7 11 9 30/30 Read block 335 len 4096 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:05.846639422Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1632} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 185 185 185 T T T 1 020-039 112 112 112 188 188 188 F F F 2 040-059 112 112 112 188 188 188 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 110 110 110 183 183 183 F F F 6 120-139 111 111 111 184 184 184 T T T 7 140-159 112 112 112 187 187 187 F F F 8 160-179 106 106 106 172 172 172 T T T 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 109 109 109 178 178 178 F F F 12 240-259 112 112 112 186 186 186 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 112 112 112 188 188 188 T T T 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 111 111 111 184 184 184 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 111 111 111 184 184 184 F F F 23 460-479 110 110 110 183 183 183 F F F 24 480-499 100 100 100 164 164 164 T T T 25 500-519 110 110 110 183 183 183 F F F 26 520-539 112 112 112 187 187 187 T T T 27 540-559 112 112 112 187 187 187 T T T 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 112, Max flush: 188 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 189 189 189 F F F 1 020-039 112 112 112 188 188 188 F F F 2 040-059 112 112 112 188 188 188 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 110 110 110 183 183 183 F F F 6 120-139 113 113 113 189 189 189 F F F 7 140-159 112 112 112 187 187 187 F F F 8 160-179 113 113 113 189 189 189 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 109 109 109 178 178 178 F F F 12 240-259 112 112 112 186 186 186 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 113 113 113 189 189 189 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 111 111 111 184 184 184 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 111 111 111 184 184 184 F F F 23 460-479 110 110 110 183 183 183 F F F 24 480-499 113 113 113 189 189 189 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 113 113 113 189 189 189 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 113, Max flush: 189 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.104244309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:10.147319738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.147776549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.180530394Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.1812886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.181321432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.181335379Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.181349156Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} {"msg":"Crucible 1ef7ae52-e656-49da-8330-dfeec4044bcf has session id: 8696bdd0-5d02-42e4-905a-808dac9ff307","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182014152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Upstairs opts: Upstairs UUID: 1ef7ae52-e656-49da-8330-dfeec4044bcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182056663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Crucible stats registered with UUID: 1ef7ae52-e656-49da-8330-dfeec4044bcf","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.18207289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:10.182085947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.18253117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182558294Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} {"msg":"1ef7ae52-e656-49da-8330-dfeec4044bcf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.18262484Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182642196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182664581Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182690106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182705082Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.18271828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182732137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182744984Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182758051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182775397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"ds_connection connected from Ok(127.0.0.1:50859)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.182889663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"ds_connection connected from Ok(127.0.0.1:59018)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.183018315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"ds_connection connected from Ok(127.0.0.1:55035)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.183148877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.183719213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.183895675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.183974038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.184309035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.184349686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.184603202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.184940648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.184966553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[0]R flush_numbers[0..12]: [189, 188, 188, 175, 173, 183, 189, 187, 189, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185203951Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[0]R generation[0..12]: [113, 112, 112, 108, 107, 110, 113, 112, 113, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185232775Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185249711Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[1]R flush_numbers[0..12]: [189, 188, 188, 175, 173, 183, 189, 187, 189, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185267618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[1]R generation[0..12]: [113, 112, 112, 108, 107, 110, 113, 112, 113, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185282704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185296971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[2]R flush_numbers[0..12]: [189, 188, 188, 175, 173, 183, 189, 187, 189, 184, 178, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185310388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"[2]R generation[0..12]: [113, 112, 112, 108, 107, 110, 113, 112, 113, 111, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185323875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185343711Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185356968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185369905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185382473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.18539504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"":"downstairs","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185408597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185420514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"0","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185437941Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"1","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185455397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"client":"2","":"downstairs","session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185471623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185483831Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"1ef7ae52-e656-49da-8330-dfeec4044bcf is now active with session: 8696bdd0-5d02-42e4-905a-808dac9ff307"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185499467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185511305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640,"session_id":"8696bdd0-5d02-42e4-905a-808dac9ff307"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:10.185522922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 236 len 8192 data: 9 10 02/30 Read block 264 len 40960 03/30 Read block 443 len 4096 04/30 Read block 331 len 4096 05/30 Read block 582 len 20480 06/30 Write block 133 len 8192 data: 8 10 07/30 Flush 08/30 Write block 481 len 40960 data: 12 9 11 10 9 7 6 7 9 8 09/30 Write block 240 len 16384 data: 11 10 13 10 10/30 Write block 32 len 8192 data: 12 10 11/30 Write block 122 len 36864 data: 12 12 12 11 6 7 8 7 6 12/30 Read block 195 len 36864 13/30 Read block 202 len 28672 14/30 Read block 475 len 40960 15/30 Write block 598 len 4096 data: 3 16/30 Write block 84 len 32768 data: 10 14 11 11 10 11 9 8 17/30 Read block 411 len 36864 18/30 Read block 310 len 28672 19/30 Write block 265 len 36864 data: 13 12 13 14 13 11 11 12 15 20/30 Read block 344 len 12288 21/30 Write block 0 len 16384 data: 5 6 7 7 22/30 Read block 183 len 4096 23/30 Write block 53 len 4096 data: 7 24/30 Write block 266 len 12288 data: 13 14 15 25/30 Write block 104 len 4096 data: 7 26/30 Flush 27/30 Write block 394 len 20480 data: 8 8 7 8 11 28/30 Write block 29 len 4096 data: 10 29/30 Read block 537 len 32768 30/30 Read block 428 len 40960 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:10.204363211Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1640} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 114 114 114 191 191 191 T T T 2 040-059 114 114 114 191 191 191 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 114 114 114 191 191 191 F F F 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 112 112 112 187 187 187 F F F 8 160-179 113 113 113 189 189 189 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 114 114 114 190 190 190 F F F 12 240-259 114 114 114 191 191 191 F F F 13 260-279 114 114 114 191 191 191 F F F 14 280-299 113 113 113 189 189 189 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 111 111 111 184 184 184 T T T 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 111 111 111 184 184 184 F F F 23 460-479 110 110 110 183 183 183 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 113 113 113 189 189 189 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 114 114 114 191 191 191 F F F Max gen: 114, Max flush: 191 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 114 114 114 191 191 191 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 114 114 114 191 191 191 F F F 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 112 112 112 187 187 187 F F F 8 160-179 113 113 113 189 189 189 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 114 114 114 190 190 190 F F F 12 240-259 114 114 114 191 191 191 F F F 13 260-279 114 114 114 191 191 191 F F F 14 280-299 113 113 113 189 189 189 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 100 100 100 164 164 164 F F F 19 380-399 115 115 115 192 192 192 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 111 111 111 184 184 184 F F F 23 460-479 110 110 110 183 183 183 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 113 113 113 189 189 189 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 114 114 114 191 191 191 F F F Max gen: 115, Max flush: 192 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.463136526Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.5068771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.507605353Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.507649893Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.507669768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.507685075Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} {"msg":"Crucible 136268e9-05a0-4a15-b099-bef9e306c71f has session id: fb5f7a31-c05b-4afc-a6e8-45363696e809","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508256512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Upstairs opts: Upstairs UUID: 136268e9-05a0-4a15-b099-bef9e306c71f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508304131Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Crucible stats registered with UUID: 136268e9-05a0-4a15-b099-bef9e306c71f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508326886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:14.508347602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508748485Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508782028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} {"msg":"136268e9-05a0-4a15-b099-bef9e306c71f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508835456Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.50886291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508888315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.50890772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508933395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508966448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.508997801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509023945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509051329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509079413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"ds_connection connected from Ok(127.0.0.1:43684)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509107777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"ds_connection connected from Ok(127.0.0.1:57742)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509135891Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"ds_connection connected from Ok(127.0.0.1:45331)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509163745Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.509993475Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.510032727Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.510636406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.510787463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.510821386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511275567Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511317098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511379305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[0]R flush_numbers[0..12]: [191, 192, 191, 175, 191, 191, 191, 187, 189, 184, 178, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511711583Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[0]R generation[0..12]: [114, 115, 114, 108, 114, 114, 114, 112, 113, 111, 109, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511732719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511747875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[1]R flush_numbers[0..12]: [191, 192, 191, 175, 191, 191, 191, 187, 189, 184, 178, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511762142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[1]R generation[0..12]: [114, 115, 114, 108, 114, 114, 114, 112, 113, 111, 109, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511775919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511789836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[2]R flush_numbers[0..12]: [191, 192, 191, 175, 191, 191, 191, 187, 189, 184, 178, 190]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511803663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[2]R generation[0..12]: [114, 115, 114, 108, 114, 114, 114, 112, 113, 111, 109, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.51181707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511830707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:14.511843195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511860121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511872578Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511890604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"":"downstairs","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511904431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511916039Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"0","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511929296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"1","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511942333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"client":"2","":"downstairs","session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.5119554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511967318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"136268e9-05a0-4a15-b099-bef9e306c71f is now active with session: fb5f7a31-c05b-4afc-a6e8-45363696e809","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511979625Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.511991123Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648,"session_id":"fb5f7a31-c05b-4afc-a6e8-45363696e809"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:14.51200285Z",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 32768 data: 10 10 10 10 8 7 6 6 02/30 Read block 517 len 4096 03/30 Write block 332 len 16384 data: 11 9 8 7 04/30 Flush 05/30 Write block 163 len 20480 data: 9 9 7 8 7 06/30 Read block 41 len 36864 07/30 Read block 138 len 40960 08/30 Read block 13 len 8192 09/30 Read block 423 len 4096 10/30 Read block 123 len 12288 11/30 Read block 127 len 36864 12/30 Write block 85 len 20480 data: 15 12 12 11 12 13/30 Write block 287 len 40960 data: 12 11 10 9 12 11 10 14 13 14 14/30 Read block 449 len 24576 15/30 Read block 575 len 36864 16/30 Read block 141 len 32768 17/30 Write block 364 len 12288 data: 7 7 7 18/30 Read block 102 len 36864 19/30 Read block 336 len 40960 20/30 Write block 235 len 8192 data: 11 10 21/30 Write block 456 len 40960 data: 11 9 9 9 9 8 9 6 6 7 22/30 Read block 210 len 32768 23/30 Write block 42 len 8192 data: 7 5 24/30 Write block 470 len 40960 data: 9 11 11 7 8 8 9 9 9 9 25/30 Write block 259 len 28672 data: 11 10 9 10 10 12 14 26/30 Read block 47 len 8192 27/30 Write block 336 len 20480 data: 7 5 7 9 11 28/30 Read block 243 len 20480 29/30 Read block 392 len 16384 30/30 Write block 368 len 40960 data: 7 6 9 9 9 9 7 9 8 6 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 5 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:14.532811702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1648} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 114 114 114 191 191 191 T T T 3 060-079 108 108 108 175 175 175 F F F 4 080-099 114 114 114 191 191 191 T T T 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 112 112 112 187 187 187 F F F 8 160-179 113 113 113 189 189 189 T T T 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 114 114 114 190 190 190 T T T 12 240-259 114 114 114 191 191 191 T T T 13 260-279 114 114 114 191 191 191 T T T 14 280-299 113 113 113 189 189 189 T T T 15 300-319 108 108 108 177 177 177 F F F 16 320-339 116 116 116 193 193 193 T T T 17 340-359 104 104 104 170 170 170 T T T 18 360-379 100 100 100 164 164 164 T T T 19 380-399 115 115 115 192 192 192 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 111 111 111 184 184 184 T T T 23 460-479 110 110 110 183 183 183 T T T 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 113 113 113 189 189 189 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 114 114 114 191 191 191 F F F Max gen: 116, Max flush: 193 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 117 117 117 194 194 194 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 117 117 117 194 194 194 F F F 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 112 112 112 187 187 187 F F F 8 160-179 117 117 117 194 194 194 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 117 117 117 194 194 194 F F F 14 280-299 117 117 117 194 194 194 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 117 117 117 194 194 194 F F F 17 340-359 117 117 117 194 194 194 F F F 18 360-379 117 117 117 194 194 194 F F F 19 380-399 115 115 115 192 192 192 F F F 20 400-419 111 111 111 184 184 184 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 117 117 117 194 194 194 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 113 113 113 189 189 189 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 114 114 114 191 191 191 F F F Max gen: 117, Max flush: 194 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.81285177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:18.855482665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.855978138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.886841882Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.887518025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.887554117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.887568664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.887582861Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} {"msg":"Crucible 67a01093-93c6-4772-ac8f-38e557b2d131 has session id: f14968b8-dbc7-44c6-b260-98093c649390","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888199098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Upstairs opts: Upstairs UUID: 67a01093-93c6-4772-ac8f-38e557b2d131, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.88823309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Crucible stats registered with UUID: 67a01093-93c6-4772-ac8f-38e557b2d131","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888248297Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:18.888264174Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888687482Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888711557Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} {"msg":"67a01093-93c6-4772-ac8f-38e557b2d131 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888778702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888796099Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888818414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888832421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888852446Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888877601Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888896767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888910244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888931859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.888963112Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"ds_connection connected from Ok(127.0.0.1:62993)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889011592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"ds_connection connected from Ok(127.0.0.1:63472)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889240372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"ds_connection connected from Ok(127.0.0.1:48184)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889284273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889815548Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889855179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.889885923Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.890486832Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.890659285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.890685559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891081473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891179882Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[0]R flush_numbers[0..12]: [191, 192, 194, 175, 194, 191, 191, 187, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891304635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[0]R generation[0..12]: [114, 115, 117, 108, 117, 114, 114, 112, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891325451Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891343497Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[1]R flush_numbers[0..12]: [191, 192, 194, 175, 194, 191, 191, 187, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891357434Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[1]R generation[0..12]: [114, 115, 117, 108, 117, 114, 114, 112, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891370781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891384308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[2]R flush_numbers[0..12]: [191, 192, 194, 175, 194, 191, 191, 187, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891397455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[2]R generation[0..12]: [114, 115, 117, 108, 117, 114, 114, 112, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891410592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891423999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891436127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891449044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:18.891461741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891482877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"":"downstairs","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891495824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891507261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"0","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891520328Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"1","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891535095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"client":"2","":"downstairs","session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891548212Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.89156012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"67a01093-93c6-4772-ac8f-38e557b2d131 is now active with session: f14968b8-dbc7-44c6-b260-98093c649390","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891572527Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891584175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656,"session_id":"f14968b8-dbc7-44c6-b260-98093c649390"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:18.891595792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 4096 data: 11 02/30 Read block 344 len 24576 03/30 Flush 04/30 Write block 469 len 12288 data: 8 10 12 05/30 Read block 204 len 8192 06/30 Read block 377 len 8192 07/30 Read block 439 len 12288 08/30 Read block 464 len 40960 09/30 Flush 10/30 Write block 268 len 40960 data: 16 14 12 12 13 16 14 12 11 11 11/30 Write block 278 len 20480 data: 13 12 14 16 15 12/30 Write block 80 len 36864 data: 7 6 9 9 11 16 13 13 12 13/30 Read block 337 len 40960 14/30 Read block 279 len 32768 15/30 Flush 16/30 Write block 541 len 36864 data: 18 17 15 16 13 13 9 10 7 17/30 Read block 278 len 8192 18/30 Write block 335 len 28672 data: 8 8 6 8 10 12 9 19/30 Flush 20/30 Write block 390 len 40960 data: 8 8 8 6 9 9 8 9 12 12 21/30 Write block 592 len 16384 data: 7 7 5 5 22/30 Read block 338 len 20480 23/30 Read block 119 len 12288 24/30 Read block 403 len 4096 25/30 Read block 561 len 28672 26/30 Read block 247 len 24576 27/30 Read block 141 len 28672 28/30 Write block 146 len 36864 data: 12 12 8 9 9 8 7 7 7 29/30 Write block 411 len 8192 data: 12 11 30/30 Write block 82 len 24576 data: 10 10 12 17 14 14 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:18.913859189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1656} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 117 117 117 194 194 194 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 118 118 118 197 197 197 T T T 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 112 112 112 187 187 187 T T T 8 160-179 117 117 117 194 194 194 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 198 198 198 F F F 18 360-379 117 117 117 194 194 194 F F F 19 380-399 115 115 115 192 192 192 T T T 20 400-419 118 118 118 195 195 195 T T T 21 420-439 112 112 112 187 187 187 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 118 118 118 196 196 196 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 118 118 118 198 198 198 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 114 114 114 191 191 191 T T T Max gen: 118, Max flush: 198 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 117 117 117 194 194 194 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 119 119 119 199 199 199 F F F 5 100-119 114 114 114 191 191 191 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 119 119 119 199 199 199 F F F 8 160-179 117 117 117 194 194 194 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 109 109 109 178 178 178 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 108 108 108 177 177 177 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 198 198 198 F F F 18 360-379 117 117 117 194 194 194 F F F 19 380-399 119 119 119 199 199 199 F F F 20 400-419 119 119 119 199 199 199 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 118 118 118 196 196 196 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 113 113 113 189 189 189 F F F 27 540-559 118 118 118 198 198 198 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 119 119 119 199 199 199 F F F Max gen: 119, Max flush: 199 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.190405675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.234431418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235147263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235191124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235206021Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235220797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} {"msg":"Crucible 3d01851c-c552-43bb-8e76-07d2763ffea3 has session id: e2cd5d67-971d-4747-94a7-2ec0c27273d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235756411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Upstairs opts: Upstairs UUID: 3d01851c-c552-43bb-8e76-07d2763ffea3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.235794933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Crucible stats registered with UUID: 3d01851c-c552-43bb-8e76-07d2763ffea3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23581037Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:23.235823737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236215332Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236240726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} {"msg":"3d01851c-c552-43bb-8e76-07d2763ffea3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236289856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23631602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236338835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236356291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236377687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236401162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236426856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236448021Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236498461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236530774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61902)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23654767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54938)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.236568815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48446)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23659323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.237327051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.237356884Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.237383479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.237955875Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238094635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238125748Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238461385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238710531Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[0]R flush_numbers[0..12]: [191, 192, 194, 175, 199, 191, 191, 199, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238786225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[0]R generation[0..12]: [114, 115, 117, 108, 119, 114, 114, 119, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238813049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238829945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[1]R flush_numbers[0..12]: [191, 192, 194, 175, 199, 191, 191, 199, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238844372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[1]R generation[0..12]: [114, 115, 117, 108, 119, 114, 114, 119, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238858739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238872716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[2]R flush_numbers[0..12]: [191, 192, 194, 175, 199, 191, 191, 199, 194, 184, 178, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238886533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[2]R generation[0..12]: [114, 115, 117, 108, 119, 114, 114, 119, 117, 111, 109, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23890004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238920116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238943391Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.238977104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23899421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239016455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"":"downstairs","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239032012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239044709Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"0","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239057906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"1","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239071403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"client":"2","":"downstairs","session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23908437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239097078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"3d01851c-c552-43bb-8e76-07d2763ffea3 is now active with session: e2cd5d67-971d-4747-94a7-2ec0c27273d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239109715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.239121402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664,"session_id":"e2cd5d67-971d-4747-94a7-2ec0c27273d6"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:23.23913327Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 16384 data: 7 5 6 4 02/30 Write block 156 len 40960 data: 11 12 12 16 12 12 12 10 10 8 03/30 Write block 46 len 12288 data: 8 7 8 04/30 Read block 231 len 8192 05/30 Write block 192 len 8192 data: 13 9 06/30 Read block 175 len 16384 07/30 Write block 540 len 40960 data: 17 19 18 16 17 14 14 10 11 8 08/30 Write block 166 len 24576 data: 9 8 8 9 9 10 09/30 Read block 518 len 40960 10/30 Write block 514 len 4096 data: 7 11/30 Write block 202 len 4096 data: 13 12/30 Read block 420 len 36864 13/30 Read block 94 len 16384 14/30 Read block 290 len 20480 15/30 Flush 16/30 Write block 314 len 8192 data: 8 8 17/30 Read block 341 len 8192 18/30 Read block 120 len 40960 19/30 Write block 524 len 16384 data: 9 12 12 8 20/30 Read block 292 len 20480 21/30 Write block 114 len 12288 data: 7 9 9 22/30 Read block 292 len 8192 23/30 Write block 542 len 8192 data: 19 17 24/30 Write block 182 len 28672 data: 11 12 13 13 14 14 13 25/30 Read block 228 len 36864 26/30 Write block 474 len 20480 data: 9 9 10 10 10 27/30 Read block 253 len 40960 28/30 Write block 153 len 8192 data: 8 8 29/30 Write block 467 len 16384 data: 11 11 9 11 30/30 Write block 192 len 16384 data: 14 10 8 9 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:23.257260433Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1664} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 120 120 120 200 200 200 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 119 119 119 199 199 199 F F F 5 100-119 114 114 114 191 191 191 T T T 6 120-139 114 114 114 191 191 191 F F F 7 140-159 120 120 120 200 200 200 T T T 8 160-179 120 120 120 200 200 200 F F F 9 180-199 120 120 120 200 200 200 T T T 10 200-219 120 120 120 200 200 200 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 108 108 108 177 177 177 T T T 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 198 198 198 F F F 18 360-379 117 117 117 194 194 194 F F F 19 380-399 119 119 119 199 199 199 F F F 20 400-419 119 119 119 199 199 199 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 118 118 118 196 196 196 T T T 24 480-499 114 114 114 191 191 191 F F F 25 500-519 120 120 120 200 200 200 F F F 26 520-539 113 113 113 189 189 189 T T T 27 540-559 120 120 120 200 200 200 T T T 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 120, Max flush: 200 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 191 191 191 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 120 120 120 200 200 200 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 119 119 119 199 199 199 F F F 5 100-119 121 121 121 201 201 201 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 121 121 121 201 201 201 F F F 8 160-179 120 120 120 200 200 200 F F F 9 180-199 121 121 121 201 201 201 F F F 10 200-219 120 120 120 200 200 200 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 198 198 198 F F F 18 360-379 117 117 117 194 194 194 F F F 19 380-399 119 119 119 199 199 199 F F F 20 400-419 119 119 119 199 199 199 F F F 21 420-439 112 112 112 187 187 187 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 121 121 121 201 201 201 F F F 24 480-499 114 114 114 191 191 191 F F F 25 500-519 120 120 120 200 200 200 F F F 26 520-539 121 121 121 201 201 201 F F F 27 540-559 121 121 121 201 201 201 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 121, Max flush: 201 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.509243248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:27.551884202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.552545358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58411458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.585039949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58508295Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.585100686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.585118322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} {"msg":"Crucible 1bc4b5d3-008a-42ec-ab52-d5c0b55a38fa has session id: f01755c4-62b0-49ff-a667-cc2209bbbe95","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58572905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Upstairs opts: Upstairs UUID: 1bc4b5d3-008a-42ec-ab52-d5c0b55a38fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58577418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Crucible stats registered with UUID: 1bc4b5d3-008a-42ec-ab52-d5c0b55a38fa","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.585797855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:27.585818471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586261455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586289009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} {"msg":"1bc4b5d3-008a-42ec-ab52-d5c0b55a38fa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586355954Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.5863747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586401514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586426069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586452733Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586479757Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586544303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586576286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586623656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586658498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"ds_connection connected from Ok(127.0.0.1:47654)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586685713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"ds_connection connected from Ok(127.0.0.1:51664)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586712607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"ds_connection connected from Ok(127.0.0.1:46797)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.586733832Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.587404777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.587433841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.587587237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.588151315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.588191956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.588456109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.588750225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58886571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[0]R flush_numbers[0..12]: [191, 192, 200, 175, 199, 201, 191, 201, 200, 201, 200, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.588985834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[0]R generation[0..12]: [114, 115, 120, 108, 119, 121, 114, 121, 120, 121, 120, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589030765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58905083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[1]R flush_numbers[0..12]: [191, 192, 200, 175, 199, 201, 191, 201, 200, 201, 200, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589065957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[1]R generation[0..12]: [114, 115, 120, 108, 119, 121, 114, 121, 120, 121, 120, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589079894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589094061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[2]R flush_numbers[0..12]: [191, 192, 200, 175, 199, 201, 191, 201, 200, 201, 200, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589107798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[2]R generation[0..12]: [114, 115, 120, 108, 119, 121, 114, 121, 120, 121, 120, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589121035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589140731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589153178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589165965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589178553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.58919149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"":"downstairs","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589204277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589215755Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"0","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589228602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"1","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589241329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"client":"2","":"downstairs","session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589255686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589267813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"1bc4b5d3-008a-42ec-ab52-d5c0b55a38fa is now active with session: f01755c4-62b0-49ff-a667-cc2209bbbe95","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589280341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589291938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672,"session_id":"f01755c4-62b0-49ff-a667-cc2209bbbe95"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:27.589303645Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 4096 data: 13 02/30 Read block 280 len 24576 03/30 Read block 305 len 8192 04/30 Read block 117 len 4096 05/30 Write block 422 len 28672 data: 7 8 11 9 10 12 9 06/30 Write block 482 len 28672 data: 10 12 11 10 8 7 8 07/30 Read block 374 len 36864 08/30 Read block 203 len 12288 09/30 Read block 137 len 36864 10/30 Read block 404 len 40960 11/30 Write block 351 len 36864 data: 7 7 8 6 7 7 6 6 7 12/30 Write block 542 len 40960 data: 20 18 18 15 15 11 12 9 8 6 13/30 Write block 484 len 36864 data: 12 11 9 8 9 10 9 8 8 14/30 Read block 68 len 28672 15/30 Read block 204 len 8192 16/30 Write block 206 len 8192 data: 14 14 17/30 Write block 394 len 4096 data: 10 18/30 Write block 426 len 20480 data: 11 13 10 8 8 19/30 Read block 321 len 28672 20/30 Read block 246 len 4096 21/30 Write block 504 len 12288 data: 12 12 11 22/30 Write block 372 len 28672 data: 10 10 8 10 9 7 5 23/30 Read block 451 len 36864 24/30 Flush 25/30 Write block 169 len 16384 data: 10 10 11 10 26/30 Write block 411 len 28672 data: 13 12 7 9 8 6 8 27/30 Read block 590 len 16384 28/30 Read block 2 len 12288 29/30 Read block 87 len 16384 30/30 Read block 148 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:27.609743128Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1672} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 202 202 202 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 120 120 120 200 200 200 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 119 119 119 199 199 199 F F F 5 100-119 121 121 121 201 201 201 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 121 121 121 201 201 201 F F F 8 160-179 120 120 120 200 200 200 T T T 9 180-199 121 121 121 201 201 201 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 122 122 122 202 202 202 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 122 122 122 202 202 202 F F F 20 400-419 119 119 119 199 199 199 T T T 21 420-439 122 122 122 202 202 202 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 121 121 121 201 201 201 F F F 24 480-499 122 122 122 202 202 202 F F F 25 500-519 122 122 122 202 202 202 F F F 26 520-539 121 121 121 201 201 201 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 122, Max flush: 202 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 202 202 202 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 120 120 120 200 200 200 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 119 119 119 199 199 199 F F F 5 100-119 121 121 121 201 201 201 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 121 121 121 201 201 201 F F F 8 160-179 123 123 123 203 203 203 F F F 9 180-199 121 121 121 201 201 201 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 118 118 118 197 197 197 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 122 122 122 202 202 202 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 122 122 122 202 202 202 F F F 20 400-419 123 123 123 203 203 203 F F F 21 420-439 122 122 122 202 202 202 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 121 121 121 201 201 201 F F F 24 480-499 122 122 122 202 202 202 F F F 25 500-519 122 122 122 202 202 202 F F F 26 520-539 121 121 121 201 201 201 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 123, Max flush: 203 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.867631352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.911056096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.911726471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.911768721Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.911784498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.911799815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} {"msg":"Crucible 0f1b481e-8b37-41f0-b0ac-5be54020ca36 has session id: 88c956e7-5fc7-4b7c-a364-a4a78e71d91b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.912316543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Upstairs opts: Upstairs UUID: 0f1b481e-8b37-41f0-b0ac-5be54020ca36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.912363302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Crucible stats registered with UUID: 0f1b481e-8b37-41f0-b0ac-5be54020ca36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.912381229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:31.912394566Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913052793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913134775Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} {"msg":"0f1b481e-8b37-41f0-b0ac-5be54020ca36 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.9131591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913174107Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.91320402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913231105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913259008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913279134Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.91329985Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913322645Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913347859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913374763Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37981)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913402177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48528)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913431511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47347)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.913481211Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914194556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914235367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914299813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914828289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914855963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.914877158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915459962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915485546Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[0]R flush_numbers[0..12]: [202, 192, 200, 175, 199, 201, 191, 201, 203, 201, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915591803Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[0]R generation[0..12]: [122, 115, 120, 108, 119, 121, 114, 121, 123, 121, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915611629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915626436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[1]R flush_numbers[0..12]: [202, 192, 200, 175, 199, 201, 191, 201, 203, 201, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915640203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[1]R generation[0..12]: [122, 115, 120, 108, 119, 121, 114, 121, 123, 121, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.91565384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915667357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[2]R flush_numbers[0..12]: [202, 192, 200, 175, 199, 201, 191, 201, 203, 201, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915680594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[2]R generation[0..12]: [122, 115, 120, 108, 119, 121, 114, 121, 123, 121, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915693661Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915707048Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915719666Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915732293Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915747829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915773064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"":"downstairs","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915787551Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915799618Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"0","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915812295Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"1","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915824813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"client":"2","":"downstairs","session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.91583735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915849198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"0f1b481e-8b37-41f0-b0ac-5be54020ca36 is now active with session: 88c956e7-5fc7-4b7c-a364-a4a78e71d91b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915861575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.915873052Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680,"session_id":"88c956e7-5fc7-4b7c-a364-a4a78e71d91b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:31.91588488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 16384 data: 7 11 10 9 02/30 Read block 348 len 36864 03/30 Write block 462 len 28672 data: 10 7 7 8 9 12 12 04/30 Read block 508 len 24576 05/30 Write block 96 len 4096 data: 3 06/30 Read block 537 len 4096 07/30 Write block 154 len 8192 data: 9 8 08/30 Read block 219 len 32768 09/30 Write block 501 len 28672 data: 8 11 11 13 13 12 9 10/30 Read block 25 len 20480 11/30 Write block 85 len 36864 data: 18 15 15 13 13 10 9 6 7 12/30 Write block 342 len 8192 data: 11 12 13/30 Write block 260 len 4096 data: 11 14/30 Write block 103 len 32768 data: 9 8 9 10 10 10 9 10 15/30 Write block 288 len 16384 data: 12 11 10 13 16/30 Write block 190 len 28672 data: 13 11 15 11 9 10 6 17/30 Read block 233 len 8192 18/30 Read block 386 len 24576 19/30 Flush 20/30 Write block 57 len 4096 data: 9 21/30 Flush 22/30 Write block 4 len 16384 data: 8 10 11 10 23/30 Read block 87 len 16384 24/30 Read block 548 len 8192 25/30 Read block 536 len 16384 26/30 Flush 27/30 Write block 336 len 32768 data: 9 7 9 11 13 10 12 13 28/30 Write block 337 len 4096 data: 8 29/30 Write block 482 len 32768 data: 11 13 13 12 10 9 10 11 30/30 Write block 532 len 8192 data: 13 14 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:31.933258676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1680} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 206 206 206 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 124 124 124 205 205 205 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 124 124 124 204 204 204 F F F 5 100-119 124 124 124 204 204 204 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 124 124 124 204 204 204 F F F 8 160-179 123 123 123 203 203 203 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 118 118 118 198 198 198 T T T 17 340-359 124 124 124 204 204 204 T T T 18 360-379 122 122 122 202 202 202 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 123 123 123 203 203 203 F F F 21 420-439 122 122 122 202 202 202 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 124 124 124 204 204 204 F F F 24 480-499 122 122 122 202 202 202 T T T 25 500-519 124 124 124 204 204 204 F F F 26 520-539 121 121 121 201 201 201 T T T 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 124, Max flush: 206 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 206 206 206 F F F 1 020-039 115 115 115 192 192 192 F F F 2 040-059 124 124 124 205 205 205 F F F 3 060-079 108 108 108 175 175 175 F F F 4 080-099 124 124 124 204 204 204 F F F 5 100-119 124 124 124 204 204 204 F F F 6 120-139 114 114 114 191 191 191 F F F 7 140-159 124 124 124 204 204 204 F F F 8 160-179 123 123 123 203 203 203 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 117 117 117 194 194 194 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 125 125 125 207 207 207 F F F 18 360-379 122 122 122 202 202 202 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 123 123 123 203 203 203 F F F 21 420-439 122 122 122 202 202 202 F F F 22 440-459 117 117 117 194 194 194 F F F 23 460-479 124 124 124 204 204 204 F F F 24 480-499 125 125 125 207 207 207 F F F 25 500-519 124 124 124 204 204 204 F F F 26 520-539 125 125 125 207 207 207 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 120 120 120 200 200 200 F F F Max gen: 125, Max flush: 207 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.171498859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:36.213807534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.214231912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.245272528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.245905511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.245946692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.245963418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.245981665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} {"msg":"Crucible 2717a7c7-a5a7-4700-a1c9-fbf503d94448 has session id: 8dcb5001-bc9f-4f59-b233-ea200376a8d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.246563828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Upstairs opts: Upstairs UUID: 2717a7c7-a5a7-4700-a1c9-fbf503d94448, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.246618657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Crucible stats registered with UUID: 2717a7c7-a5a7-4700-a1c9-fbf503d94448","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.246642471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:36.246664577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247053762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247087005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} {"msg":"2717a7c7-a5a7-4700-a1c9-fbf503d94448 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247145053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247172427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247197991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247213798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247228775Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247242461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247257028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247270695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247318975Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247354777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49562)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247381062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45516)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247406296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62691)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.247436829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248096966Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248124301Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248407249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248856072Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248898063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.248929566Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249486525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.24951264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[0]R flush_numbers[0..12]: [206, 192, 205, 175, 204, 204, 191, 204, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.2495556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[0]R generation[0..12]: [124, 115, 124, 108, 124, 124, 114, 124, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249573396Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249587624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[1]R flush_numbers[0..12]: [206, 192, 205, 175, 204, 204, 191, 204, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.24960127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[1]R generation[0..12]: [124, 115, 124, 108, 124, 124, 114, 124, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249615478Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249633623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[2]R flush_numbers[0..12]: [206, 192, 205, 175, 204, 204, 191, 204, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249647491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[2]R generation[0..12]: [124, 115, 124, 108, 124, 124, 114, 124, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249661008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249674525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249686862Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249699419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249711597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:36.249723974Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"":"downstairs","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249781402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249794189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"0","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249807126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"1","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249819523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"client":"2","":"downstairs","session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.24983255Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249844648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"2717a7c7-a5a7-4700-a1c9-fbf503d94448 is now active with session: 8dcb5001-bc9f-4f59-b233-ea200376a8d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249857075Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249871402Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688,"session_id":"8dcb5001-bc9f-4f59-b233-ea200376a8d6"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:36.249883759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 4096 data: 9 02/30 Read block 48 len 12288 03/30 Write block 364 len 36864 data: 8 8 8 7 8 7 10 10 11 04/30 Write block 33 len 24576 data: 11 14 12 12 10 11 05/30 Write block 85 len 24576 data: 19 16 16 14 14 11 06/30 Read block 132 len 16384 07/30 Write block 97 len 40960 data: 5 7 5 7 8 9 10 9 10 11 08/30 Read block 156 len 12288 09/30 Read block 103 len 16384 10/30 Write block 580 len 40960 data: 10 10 9 6 8 6 7 5 6 5 11/30 Flush 12/30 Write block 241 len 16384 data: 11 14 11 9 13/30 Read block 62 len 24576 14/30 Read block 345 len 40960 15/30 Read block 233 len 40960 16/30 Write block 421 len 40960 data: 9 8 9 12 10 12 14 11 9 9 17/30 Write block 111 len 40960 data: 12 11 8 8 10 10 7 9 11 12 18/30 Write block 453 len 40960 data: 11 12 10 12 10 10 10 10 9 11 19/30 Read block 587 len 16384 20/30 Read block 387 len 32768 21/30 Read block 576 len 24576 22/30 Write block 349 len 40960 data: 9 9 8 8 9 7 8 8 7 7 23/30 Write block 513 len 40960 data: 6 8 7 7 8 5 6 8 10 10 24/30 Read block 70 len 28672 25/30 Read block 157 len 20480 26/30 Flush 27/30 Write block 154 len 4096 data: 10 28/30 Read block 309 len 12288 29/30 Read block 81 len 4096 30/30 Read block 14 len 36864 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:36.272056286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1688} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 206 206 206 F F F 1 020-039 126 126 126 208 208 208 F F F 2 040-059 124 124 124 205 205 205 F F F 3 060-079 126 126 126 208 208 208 F F F 4 080-099 126 126 126 208 208 208 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 126 126 126 209 209 209 F F F 7 140-159 124 124 124 204 204 204 T T T 8 160-179 123 123 123 203 203 203 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 126 126 126 209 209 209 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 126 126 126 209 209 209 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 123 123 123 203 203 203 F F F 21 420-439 126 126 126 209 209 209 F F F 22 440-459 126 126 126 209 209 209 F F F 23 460-479 126 126 126 209 209 209 F F F 24 480-499 125 125 125 207 207 207 F F F 25 500-519 126 126 126 209 209 209 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 126, Max flush: 209 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 206 206 206 F F F 1 020-039 126 126 126 208 208 208 F F F 2 040-059 124 124 124 205 205 205 F F F 3 060-079 126 126 126 208 208 208 F F F 4 080-099 126 126 126 208 208 208 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 126 126 126 209 209 209 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 123 123 123 203 203 203 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 117 117 117 194 194 194 F F F 12 240-259 126 126 126 209 209 209 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 126 126 126 209 209 209 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 123 123 123 203 203 203 F F F 21 420-439 126 126 126 209 209 209 F F F 22 440-459 126 126 126 209 209 209 F F F 23 460-479 126 126 126 209 209 209 F F F 24 480-499 125 125 125 207 207 207 F F F 25 500-519 126 126 126 209 209 209 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 127, Max flush: 210 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.523666573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.569160308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570097505Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570141696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570156812Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570175468Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} {"msg":"Crucible 52c2a018-b6dd-4216-a8a6-62dda9a8ead9 has session id: a9ff0e2d-16c5-4651-9836-867d7da8edc8","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570778318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Upstairs opts: Upstairs UUID: 52c2a018-b6dd-4216-a8a6-62dda9a8ead9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.57081373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Crucible stats registered with UUID: 52c2a018-b6dd-4216-a8a6-62dda9a8ead9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.570829127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:40.570845353Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571236549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571259204Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} {"msg":"52c2a018-b6dd-4216-a8a6-62dda9a8ead9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571309713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571336237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571359612Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571373899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571390715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.57141405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571441344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571471608Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571498782Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571523196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58028)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571567957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49810)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571604259Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35570)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.571795857Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.572232553Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.572386089Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.572542266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.572949867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.572978401Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573253352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573535731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573653545Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[0]R flush_numbers[0..12]: [206, 208, 205, 208, 208, 209, 209, 210, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.57390504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[0]R generation[0..12]: [124, 126, 124, 126, 126, 126, 126, 127, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573928226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573943732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[1]R flush_numbers[0..12]: [206, 208, 205, 208, 208, 209, 209, 210, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573957389Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[1]R generation[0..12]: [124, 126, 124, 126, 126, 126, 126, 127, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573970626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.573984283Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[2]R flush_numbers[0..12]: [206, 208, 205, 208, 208, 209, 209, 210, 203, 204, 202, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.57399777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[2]R generation[0..12]: [124, 126, 124, 126, 126, 126, 126, 127, 123, 124, 122, 117]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574011248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574024665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574037182Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Generation requested: 128 >= found:128",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574056418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574069005Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574081492Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"":"downstairs","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574094119Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574106697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"0","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574122403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"1","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574135281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"client":"2","":"downstairs","session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574147978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574159575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"52c2a018-b6dd-4216-a8a6-62dda9a8ead9 is now active with session: a9ff0e2d-16c5-4651-9836-867d7da8edc8","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:43:40.574171853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.5741836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696,"session_id":"a9ff0e2d-16c5-4651-9836-867d7da8edc8"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:40.574195437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 8192 data: 13 12 02/30 Write block 134 len 16384 data: 11 11 10 7 03/30 Write block 230 len 20480 data: 8 10 10 10 9 04/30 Read block 250 len 32768 05/30 Flush 06/30 Write block 15 len 4096 data: 11 07/30 Read block 309 len 32768 08/30 Write block 79 len 32768 data: 5 8 7 11 11 13 20 17 09/30 Write block 121 len 8192 data: 12 13 10/30 Read block 241 len 12288 11/30 Read block 447 len 20480 12/30 Flush 13/30 Write block 458 len 4096 data: 11 14/30 Write block 170 len 24576 data: 11 12 11 9 10 10 15/30 Write block 67 len 36864 data: 10 9 11 9 8 9 11 8 7 16/30 Flush 17/30 Write block 137 len 12288 data: 8 7 10 18/30 Read block 259 len 32768 19/30 Write block 346 len 36864 data: 13 11 8 10 10 9 9 10 8 20/30 Read block 439 len 32768 21/30 Read block 594 len 8192 22/30 Read block 59 len 8192 23/30 Read block 414 len 40960 24/30 Read block 361 len 24576 25/30 Flush 26/30 Flush 27/30 Write block 36 len 12288 data: 13 11 12 28/30 Write block 240 len 40960 data: 12 12 15 12 10 8 7 6 5 5 29/30 Write block 61 len 20480 data: 13 14 10 10 9 30/30 Write block 415 len 28672 data: 9 7 9 11 8 10 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:40.592423579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1696} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 212 212 212 F F F 1 020-039 126 126 126 208 208 208 T T T 2 040-059 124 124 124 205 205 205 F F F 3 060-079 128 128 128 213 213 213 T T T 4 080-099 128 128 128 212 212 212 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 128 128 128 214 214 214 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 128 128 128 213 213 213 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 128 128 128 211 211 211 F F F 12 240-259 126 126 126 209 209 209 T T T 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 128 128 128 214 214 214 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 123 123 123 203 203 203 T T T 21 420-439 126 126 126 209 209 209 T T T 22 440-459 128 128 128 213 213 213 F F F 23 460-479 126 126 126 209 209 209 F F F 24 480-499 125 125 125 207 207 207 F F F 25 500-519 126 126 126 209 209 209 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 128, Max flush: 214 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 212 212 212 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 124 124 124 205 205 205 F F F 3 060-079 129 129 129 215 215 215 F F F 4 080-099 128 128 128 212 212 212 F F F 5 100-119 126 126 126 209 209 209 F F F 6 120-139 128 128 128 214 214 214 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 128 128 128 213 213 213 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 128 128 128 211 211 211 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 124 124 124 204 204 204 F F F 15 300-319 121 121 121 201 201 201 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 128 128 128 214 214 214 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 124 124 124 204 204 204 F F F 20 400-419 129 129 129 215 215 215 F F F 21 420-439 129 129 129 215 215 215 F F F 22 440-459 128 128 128 213 213 213 F F F 23 460-479 126 126 126 209 209 209 F F F 24 480-499 125 125 125 207 207 207 F F F 25 500-519 126 126 126 209 209 209 F F F 26 520-539 126 126 126 209 209 209 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 129, Max flush: 215 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.847035035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.89123747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.891919572Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.891968311Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.891986277Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.892003664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} {"msg":"Crucible 1f5dcfec-3feb-49df-8ae2-ac2973e285d7 has session id: e5bd01bf-9b77-4cb0-a727-9c24e504f986","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.892652083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Upstairs opts: Upstairs UUID: 1f5dcfec-3feb-49df-8ae2-ac2973e285d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.892694374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Crucible stats registered with UUID: 1f5dcfec-3feb-49df-8ae2-ac2973e285d7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.892721018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:44.892742824Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893273249Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.89331051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} {"msg":"1f5dcfec-3feb-49df-8ae2-ac2973e285d7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893363889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893379655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893400141Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893418007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893432514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893454199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893481704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893508827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893531013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.89354477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"ds_connection connected from Ok(127.0.0.1:57719)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893646058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"ds_connection connected from Ok(127.0.0.1:38093)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.89368237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"ds_connection connected from Ok(127.0.0.1:55129)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.893833837Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.894303155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.894340707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.894460401Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.894941697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.894985038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.89501846Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895516233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895632997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[0]R flush_numbers[0..12]: [212, 215, 205, 215, 212, 209, 214, 210, 213, 204, 202, 211]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895663051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[0]R generation[0..12]: [128, 129, 124, 129, 128, 126, 128, 127, 128, 124, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895679587Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895694064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[1]R flush_numbers[0..12]: [212, 215, 205, 215, 212, 209, 214, 210, 213, 204, 202, 211]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895708011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[1]R generation[0..12]: [128, 129, 124, 129, 128, 126, 128, 127, 128, 124, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895721688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895745853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[2]R flush_numbers[0..12]: [212, 215, 205, 215, 212, 209, 214, 210, 213, 204, 202, 211]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895771317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[2]R generation[0..12]: [128, 129, 124, 129, 128, 126, 128, 127, 128, 124, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895797372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895813028Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895831255Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895863827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895890202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895908948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"":"downstairs","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895922665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895936772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"0","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895960487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"1","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.895986001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"client":"2","":"downstairs","session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.896007556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.896020214Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"1f5dcfec-3feb-49df-8ae2-ac2973e285d7 is now active with session: e5bd01bf-9b77-4cb0-a727-9c24e504f986","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.896032691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.896049137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704,"session_id":"e5bd01bf-9b77-4cb0-a727-9c24e504f986"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:44.896071032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 24576 data: 10 11 11 9 11 11 02/30 Read block 303 len 16384 03/30 Write block 392 len 12288 data: 9 8 12 04/30 Flush 05/30 Write block 179 len 36864 data: 10 9 9 12 13 14 14 15 15 06/30 Flush 07/30 Write block 289 len 4096 data: 12 08/30 Write block 535 len 16384 data: 11 9 13 12 09/30 Write block 430 len 32768 data: 10 8 9 8 7 7 7 7 10/30 Write block 44 len 40960 data: 7 8 9 8 9 8 10 9 8 8 11/30 Read block 230 len 28672 12/30 Write block 402 len 32768 data: 11 11 9 7 7 8 9 11 13/30 Read block 424 len 12288 14/30 Read block 315 len 4096 15/30 Write block 298 len 20480 data: 9 7 9 10 9 16/30 Write block 77 len 8192 data: 4 5 17/30 Flush 18/30 Write block 451 len 32768 data: 8 8 12 13 11 13 11 12 19/30 Read block 435 len 40960 20/30 Write block 484 len 32768 data: 14 13 11 10 11 12 10 9 21/30 Write block 68 len 32768 data: 10 12 10 9 10 12 9 8 22/30 Read block 84 len 4096 23/30 Write block 10 len 20480 data: 12 12 14 13 11 24/30 Write block 45 len 28672 data: 9 10 9 10 9 11 10 25/30 Read block 508 len 20480 26/30 Write block 100 len 8192 data: 8 9 27/30 Write block 235 len 4096 data: 12 28/30 Read block 486 len 28672 29/30 Read block 571 len 36864 30/30 Write block 421 len 12288 data: 11 9 10 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:44.915953776Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1704} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 212 212 212 T T T 1 020-039 129 129 129 215 215 215 F F F 2 040-059 130 130 130 218 218 218 T T T 3 060-079 130 130 130 218 218 218 T T T 4 080-099 128 128 128 212 212 212 F F F 5 100-119 126 126 126 209 209 209 T T T 6 120-139 128 128 128 214 214 214 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 130 130 130 217 217 217 F F F 9 180-199 130 130 130 217 217 217 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 130 130 130 216 216 216 T T T 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 130 130 130 218 218 218 F F F 15 300-319 130 130 130 218 218 218 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 128 128 128 214 214 214 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 130 130 130 218 218 218 T T T 22 440-459 128 128 128 213 213 213 T T T 23 460-479 126 126 126 209 209 209 F F F 24 480-499 125 125 125 207 207 207 T T T 25 500-519 126 126 126 209 209 209 F F F 26 520-539 130 130 130 218 218 218 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 130, Max flush: 218 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 219 219 219 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 131 131 131 219 219 219 F F F 4 080-099 128 128 128 212 212 212 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 128 128 128 214 214 214 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 130 130 130 217 217 217 F F F 9 180-199 130 130 130 217 217 217 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 131 131 131 219 219 219 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 130 130 130 218 218 218 F F F 15 300-319 130 130 130 218 218 218 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 128 128 128 214 214 214 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 126 126 126 209 209 209 F F F 24 480-499 131 131 131 219 219 219 F F F 25 500-519 126 126 126 209 209 209 F F F 26 520-539 130 130 130 218 218 218 F F F 27 540-559 122 122 122 202 202 202 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 F F F Max gen: 131, Max flush: 219 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.178435937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.222531934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.223259987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.223304427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.223320543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22333497Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} {"msg":"Crucible d6a88948-cf27-4df2-bb85-9705f9d1aa36 has session id: a1043e73-eb77-4a72-95b2-266d63bfc6e5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.223901777Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Upstairs opts: Upstairs UUID: d6a88948-cf27-4df2-bb85-9705f9d1aa36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.223950477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Crucible stats registered with UUID: d6a88948-cf27-4df2-bb85-9705f9d1aa36","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22398261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:49.224005255Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224425534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22444582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} {"msg":"d6a88948-cf27-4df2-bb85-9705f9d1aa36 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224519884Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22453749Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224566873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22458285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224597157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224609724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224623291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224635909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224649426Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224666942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57573)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224793035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48821)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.224833466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49603)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.225066145Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.225675153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.225709506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22596948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.226418043Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.226457364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.226585806Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227069491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227103744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[0]R flush_numbers[0..12]: [219, 215, 219, 219, 212, 219, 214, 210, 217, 217, 202, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227132078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[0]R generation[0..12]: [131, 129, 131, 131, 128, 131, 128, 127, 130, 130, 122, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227157312Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227181797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[1]R flush_numbers[0..12]: [219, 215, 219, 219, 212, 219, 214, 210, 217, 217, 202, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227206102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[1]R generation[0..12]: [131, 129, 131, 131, 128, 131, 128, 127, 130, 130, 122, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227235555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22726144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[2]R flush_numbers[0..12]: [219, 215, 219, 219, 212, 219, 214, 210, 217, 217, 202, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227285115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[2]R generation[0..12]: [131, 129, 131, 131, 128, 131, 128, 127, 130, 130, 122, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227311759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:49.227339833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227405998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227437602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227464546Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.22749033Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"":"downstairs","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227516854Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227541409Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"0","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227571762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"1","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227595857Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"client":"2","":"downstairs","session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227610324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227622622Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"d6a88948-cf27-4df2-bb85-9705f9d1aa36 is now active with session: a1043e73-eb77-4a72-95b2-266d63bfc6e5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227639378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227651455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712,"session_id":"a1043e73-eb77-4a72-95b2-266d63bfc6e5"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:49.227663523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 16384 data: 10 10 13 14 02/30 Write block 14 len 4096 data: 12 03/30 Write block 345 len 12288 data: 14 14 12 04/30 Read block 381 len 16384 05/30 Read block 345 len 8192 06/30 Read block 323 len 4096 07/30 Read block 519 len 40960 08/30 Flush 09/30 Write block 507 len 32768 data: 10 9 9 9 8 7 7 9 10/30 Write block 113 len 36864 data: 9 9 11 11 8 10 12 13 13 11/30 Read block 570 len 20480 12/30 Read block 291 len 28672 13/30 Write block 473 len 8192 data: 8 10 14/30 Write block 232 len 12288 data: 12 11 10 15/30 Read block 148 len 4096 16/30 Write block 108 len 24576 data: 11 10 11 13 12 10 17/30 Read block 36 len 36864 18/30 Read block 163 len 8192 19/30 Write block 300 len 36864 data: 10 11 10 9 10 9 8 8 8 20/30 Read block 144 len 36864 21/30 Write block 91 len 28672 data: 10 7 8 6 6 4 6 22/30 Flush 23/30 Write block 126 len 4096 data: 7 24/30 Write block 353 len 40960 data: 11 9 9 9 8 8 8 7 5 5 25/30 Read block 147 len 32768 26/30 Write block 534 len 36864 data: 13 12 10 14 13 16 18 20 21 27/30 Read block 454 len 24576 28/30 Write block 586 len 4096 data: 8 29/30 Read block 300 len 32768 30/30 Read block 283 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:49.24725139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1712} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 220 220 220 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 131 131 131 219 219 219 F F F 4 080-099 132 132 132 221 221 221 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 132 132 132 221 221 221 T T T 7 140-159 127 127 127 210 210 210 F F F 8 160-179 130 130 130 217 217 217 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 132 132 132 221 221 221 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 130 130 130 218 218 218 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 132 132 132 220 220 220 T T T 18 360-379 126 126 126 208 208 208 T T T 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 132 132 132 221 221 221 F F F 24 480-499 131 131 131 219 219 219 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 130 130 130 218 218 218 T T T 27 540-559 122 122 122 202 202 202 T T T 28 560-579 106 106 106 172 172 172 F F F 29 580-599 126 126 126 208 208 208 T T T Max gen: 132, Max flush: 221 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 220 220 220 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 131 131 131 219 219 219 F F F 4 080-099 132 132 132 221 221 221 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 130 130 130 217 217 217 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 132 132 132 221 221 221 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 130 130 130 218 218 218 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 125 125 125 207 207 207 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 133 133 133 222 222 222 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 132 132 132 221 221 221 F F F 24 480-499 131 131 131 219 219 219 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 133 133 133 222 222 222 F F F 27 540-559 133 133 133 222 222 222 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 133 133 133 222 222 222 F F F Max gen: 133, Max flush: 222 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.502958558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.547571764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.548357834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.548412032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.548434807Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.548471529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} {"msg":"Crucible 3059be8e-d27a-4b51-92dc-44cdd244d117 has session id: fe1d4332-21e8-444b-979e-3cf3dac1792e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.548987477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Upstairs opts: Upstairs UUID: 3059be8e-d27a-4b51-92dc-44cdd244d117, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549045195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Crucible stats registered with UUID: 3059be8e-d27a-4b51-92dc-44cdd244d117","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549063491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:53.549077018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549570381Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549645155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} {"msg":"3059be8e-d27a-4b51-92dc-44cdd244d117 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549679047Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549705762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549745343Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549777866Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.54980638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549833284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549858259Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549879994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549903129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549927104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35809)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549951538Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42757)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549975343Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43895)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.549999428Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.550725701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.550754525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.551358783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.55141899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.551485226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552058802Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552083486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552101953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[0]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 217, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552639506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[0]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 130, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552659612Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552673859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[1]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 217, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552687556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[1]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 130, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552701113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.55271512Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[2]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 217, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552728967Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[2]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 130, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552742434Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552756221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552768898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552781725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552794233Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552816798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"":"downstairs","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552832254Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552844102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"0","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552857439Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"1","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552870286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"client":"2","":"downstairs","session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552883263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552895121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"3059be8e-d27a-4b51-92dc-44cdd244d117 is now active with session: fe1d4332-21e8-444b-979e-3cf3dac1792e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552907468Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552919045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720,"session_id":"fe1d4332-21e8-444b-979e-3cf3dac1792e"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:53.552930873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 16384 data: 9 10 11 8 02/30 Read block 356 len 12288 03/30 Flush 04/30 Write block 482 len 32768 data: 12 14 15 14 12 11 12 13 05/30 Read block 454 len 12288 06/30 Read block 485 len 24576 07/30 Read block 298 len 24576 08/30 Read block 362 len 4096 09/30 Read block 325 len 24576 10/30 Read block 426 len 16384 11/30 Read block 326 len 4096 12/30 Write block 285 len 12288 data: 11 12 13 13/30 Read block 557 len 8192 14/30 Read block 304 len 36864 15/30 Read block 290 len 16384 16/30 Read block 442 len 32768 17/30 Read block 35 len 28672 18/30 Read block 390 len 32768 19/30 Read block 488 len 32768 20/30 Read block 244 len 36864 21/30 Read block 37 len 24576 22/30 Write block 546 len 12288 data: 16 12 13 23/30 Flush 24/30 Write block 168 len 32768 data: 9 11 12 13 12 10 11 11 25/30 Flush 26/30 Write block 338 len 24576 data: 10 12 14 11 13 14 27/30 Write block 160 len 40960 data: 13 13 13 11 11 9 10 9 10 12 28/30 Write block 345 len 32768 data: 15 15 13 9 11 11 10 10 29/30 Write block 472 len 4096 data: 12 30/30 Read block 68 len 20480 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:53.574005198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1720} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 220 220 220 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 131 131 131 219 219 219 F F F 4 080-099 132 132 132 221 221 221 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 134 134 134 225 225 225 T T T 9 180-199 132 132 132 220 220 220 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 132 132 132 221 221 221 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 134 134 134 224 224 224 F F F 15 300-319 134 134 134 223 223 223 F F F 16 320-339 134 134 134 223 223 223 T T T 17 340-359 133 133 133 222 222 222 T T T 18 360-379 133 133 133 222 222 222 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 132 132 132 221 221 221 T T T 24 480-499 134 134 134 224 224 224 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 133 133 133 222 222 222 F F F 27 540-559 134 134 134 224 224 224 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 133 133 133 222 222 222 F F F Max gen: 134, Max flush: 225 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 220 220 220 F F F 1 020-039 129 129 129 215 215 215 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 131 131 131 219 219 219 F F F 4 080-099 132 132 132 221 221 221 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 135 135 135 226 226 226 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 122 122 122 202 202 202 F F F 11 220-239 132 132 132 221 221 221 F F F 12 240-259 129 129 129 215 215 215 F F F 13 260-279 124 124 124 204 204 204 F F F 14 280-299 134 134 134 224 224 224 F F F 15 300-319 134 134 134 223 223 223 F F F 16 320-339 135 135 135 226 226 226 F F F 17 340-359 135 135 135 226 226 226 F F F 18 360-379 133 133 133 222 222 222 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 224 224 224 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 133 133 133 222 222 222 F F F 27 540-559 134 134 134 224 224 224 F F F 28 560-579 106 106 106 172 172 172 F F F 29 580-599 133 133 133 222 222 222 F F F Max gen: 135, Max flush: 226 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.809082155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.853215534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.853970081Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854011962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854028118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854042605Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} {"msg":"Crucible dc2f6206-0070-40b5-8988-a016feaebfb2 has session id: eaa5b544-b51f-426d-bbeb-5464c3009ab5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854721138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Upstairs opts: Upstairs UUID: dc2f6206-0070-40b5-8988-a016feaebfb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854763659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Crucible stats registered with UUID: dc2f6206-0070-40b5-8988-a016feaebfb2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.854780195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:57.854793603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855252743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855283977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} {"msg":"dc2f6206-0070-40b5-8988-a016feaebfb2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855349502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855371187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855396982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855410889Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855424656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855437373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855466877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855481994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855506898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855533932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34920)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855580452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:50381)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.85563491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45303)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.855851223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856308435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856335419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856472629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856948996Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856977779Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.856997095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857558894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857658762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[0]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 226, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857728117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[0]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 135, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857747143Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.85776149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[1]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 226, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857775187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[1]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 135, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857788994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857802861Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[2]R flush_numbers[0..12]: [220, 215, 219, 219, 221, 221, 222, 210, 226, 220, 202, 221]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857816258Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[2]R generation[0..12]: [132, 129, 131, 131, 132, 132, 133, 127, 135, 132, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857829825Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857847671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857860708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:43:57.857873876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857891752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857904219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"":"downstairs","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857917386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857929093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"0","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.85794235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"1","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857955108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"client":"2","":"downstairs","session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857967915Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.857979713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"dc2f6206-0070-40b5-8988-a016feaebfb2 is now active with session: eaa5b544-b51f-426d-bbeb-5464c3009ab5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.85799335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.858014235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728,"session_id":"eaa5b544-b51f-426d-bbeb-5464c3009ab5"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:43:57.858031181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 370 len 12288 data: 11 11 12 02/30 Flush 03/30 Write block 277 len 40960 data: 12 14 13 15 17 16 13 11 12 13 04/30 Write block 490 len 40960 data: 11 10 9 6 9 7 9 9 8 6 05/30 Write block 88 len 40960 data: 15 15 12 11 8 9 7 7 5 7 06/30 Read block 101 len 20480 07/30 Write block 237 len 36864 data: 11 14 13 13 13 16 13 11 9 08/30 Read block 79 len 20480 09/30 Flush 10/30 Write block 523 len 28672 data: 10 10 13 13 9 10 13 11/30 Flush 12/30 Write block 471 len 12288 data: 13 13 9 13/30 Write block 29 len 8192 data: 11 10 14/30 Write block 588 len 36864 data: 7 6 5 6 8 8 6 6 4 15/30 Write block 5 len 40960 data: 11 12 11 11 9 13 13 15 14 13 16/30 Write block 76 len 32768 data: 4 5 6 6 9 8 12 12 17/30 Write block 5 len 28672 data: 12 13 12 12 10 14 14 18/30 Write block 553 len 12288 data: 7 9 9 19/30 Read block 278 len 16384 20/30 Flush 21/30 Write block 342 len 16384 data: 14 15 13 16 22/30 Write block 123 len 12288 data: 13 13 12 23/30 Read block 598 len 4096 24/30 Flush 25/30 Write block 564 len 40960 data: 9 12 13 12 10 11 8 7 8 8 26/30 Read block 420 len 40960 27/30 Write block 408 len 36864 data: 10 12 10 14 13 8 10 10 8 28/30 Write block 407 len 20480 data: 9 11 13 11 15 29/30 Read block 379 len 20480 30/30 Write block 210 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:43:57.877838401Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1728} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 136 136 136 230 230 230 F F F 4 080-099 136 136 136 230 230 230 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 136 136 136 231 231 231 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 135 135 135 226 226 226 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 122 122 122 202 202 202 T T T 11 220-239 136 136 136 228 228 228 F F F 12 240-259 136 136 136 228 228 228 F F F 13 260-279 136 136 136 228 228 228 F F F 14 280-299 136 136 136 228 228 228 F F F 15 300-319 134 134 134 223 223 223 F F F 16 320-339 135 135 135 226 226 226 F F F 17 340-359 136 136 136 231 231 231 F F F 18 360-379 136 136 136 227 227 227 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 130 130 130 218 218 218 T T T 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 136 136 136 230 230 230 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 106 106 106 172 172 172 T T T 29 580-599 136 136 136 230 230 230 F F F Max gen: 136, Max flush: 231 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 131 131 131 219 219 219 F F F 3 060-079 136 136 136 230 230 230 F F F 4 080-099 136 136 136 230 230 230 F F F 5 100-119 132 132 132 221 221 221 F F F 6 120-139 136 136 136 231 231 231 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 135 135 135 226 226 226 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 137 137 137 232 232 232 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 136 136 136 228 228 228 F F F 13 260-279 136 136 136 228 228 228 F F F 14 280-299 136 136 136 228 228 228 F F F 15 300-319 134 134 134 223 223 223 F F F 16 320-339 135 135 135 226 226 226 F F F 17 340-359 136 136 136 231 231 231 F F F 18 360-379 136 136 136 227 227 227 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 137 137 137 232 232 232 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 136 136 136 230 230 230 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 137, Max flush: 232 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.104616597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.147909819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.148671914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.148726122Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.148745758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.148764664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} {"msg":"Crucible cb39383d-445f-4b68-8cb5-47505b66739b has session id: 7cbcbb63-41e4-4e21-a683-702da423a73d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149359825Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Upstairs opts: Upstairs UUID: cb39383d-445f-4b68-8cb5-47505b66739b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149411993Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Crucible stats registered with UUID: cb39383d-445f-4b68-8cb5-47505b66739b","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149436368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:02.149455804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149861556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149894529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} {"msg":"cb39383d-445f-4b68-8cb5-47505b66739b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.149955626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.14998179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150005215Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150019462Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150033489Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150046506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150069711Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150092366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150115171Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150138436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54910)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150169299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60217)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150195094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58597)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150507656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.150910679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.15095074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.151190208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.151658167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.151686691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.151702487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152225104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152330691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 219, 230, 230, 221, 231, 210, 226, 220, 232, 228]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152393378Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[0]R generation[0..12]: [136, 136, 131, 136, 136, 132, 136, 127, 135, 132, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152420292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152436458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 219, 230, 230, 221, 231, 210, 226, 220, 232, 228]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152455344Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[1]R generation[0..12]: [136, 136, 131, 136, 136, 132, 136, 127, 135, 132, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152469641Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152483598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 219, 230, 230, 221, 231, 210, 226, 220, 232, 228]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152497575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[2]R generation[0..12]: [136, 136, 131, 136, 136, 132, 136, 127, 135, 132, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152510912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152524369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152543115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152564861Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152587525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152601202Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"":"downstairs","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.15261406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152625797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"0","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152646103Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"1","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152667308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"client":"2","":"downstairs","session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152681225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152695602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"cb39383d-445f-4b68-8cb5-47505b66739b is now active with session: 7cbcbb63-41e4-4e21-a683-702da423a73d","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152710979Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:02.152731774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736,"session_id":"7cbcbb63-41e4-4e21-a683-702da423a73d"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:44:02.15274895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 218 len 36864 data: 6 8 6 7 6 9 9 8 10 02/30 Read block 139 len 12288 03/30 Flush 04/30 Write block 208 len 20480 data: 14 10 11 11 9 05/30 Write block 302 len 32768 data: 11 10 11 10 9 9 9 8 06/30 Read block 250 len 4096 07/30 Write block 532 len 16384 data: 14 15 14 13 08/30 Read block 129 len 16384 09/30 Read block 59 len 4096 10/30 Flush 11/30 Write block 496 len 36864 data: 10 10 9 7 8 9 12 12 14 12/30 Write block 170 len 20480 data: 13 14 13 11 12 13/30 Read block 228 len 24576 14/30 Write block 278 len 40960 data: 15 14 16 18 17 14 12 13 14 14 15/30 Write block 53 len 4096 data: 9 16/30 Write block 62 len 8192 data: 15 11 17/30 Read block 560 len 24576 18/30 Flush 19/30 Write block 551 len 32768 data: 7 7 8 10 10 9 10 10 20/30 Read block 346 len 36864 21/30 Write block 345 len 40960 data: 17 16 14 10 12 12 11 11 12 10 22/30 Write block 498 len 12288 data: 10 8 9 23/30 Write block 118 len 32768 data: 11 13 14 14 14 14 14 13 24/30 Write block 338 len 16384 data: 11 13 15 12 25/30 Read block 109 len 20480 26/30 Read block 90 len 36864 27/30 Read block 67 len 32768 28/30 Write block 216 len 20480 data: 6 6 7 9 7 29/30 Read block 305 len 40960 30/30 Write block 280 len 36864 data: 17 19 18 15 13 14 15 15 13 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:02.172964571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1736} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 138 138 138 235 235 235 F F F 4 080-099 136 136 136 230 230 230 F F F 5 100-119 132 132 132 221 221 221 T T T 6 120-139 136 136 136 231 231 231 T T T 7 140-159 127 127 127 210 210 210 F F F 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 138 138 138 234 234 234 T T T 11 220-239 138 138 138 233 233 233 T T T 12 240-259 136 136 136 228 228 228 F F F 13 260-279 138 138 138 235 235 235 F F F 14 280-299 138 138 138 235 235 235 T T T 15 300-319 138 138 138 234 234 234 F F F 16 320-339 135 135 135 226 226 226 T T T 17 340-359 136 136 136 231 231 231 T T T 18 360-379 136 136 136 227 227 227 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 137 137 137 232 232 232 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 136 136 136 230 230 230 F F F 24 480-499 138 138 138 235 235 235 T T T 25 500-519 138 138 138 235 235 235 T T T 26 520-539 138 138 138 234 234 234 F F F 27 540-559 136 136 136 230 230 230 T T T 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 138, Max flush: 235 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 138 138 138 235 235 235 F F F 4 080-099 136 136 136 230 230 230 F F F 5 100-119 139 139 139 236 236 236 F F F 6 120-139 139 139 139 236 236 236 F F F 7 140-159 127 127 127 210 210 210 F F F 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 139 139 139 236 236 236 F F F 11 220-239 139 139 139 236 236 236 F F F 12 240-259 136 136 136 228 228 228 F F F 13 260-279 138 138 138 235 235 235 F F F 14 280-299 139 139 139 236 236 236 F F F 15 300-319 138 138 138 234 234 234 F F F 16 320-339 139 139 139 236 236 236 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 136 136 136 227 227 227 F F F 19 380-399 130 130 130 216 216 216 F F F 20 400-419 137 137 137 232 232 232 F F F 21 420-439 131 131 131 219 219 219 F F F 22 440-459 131 131 131 219 219 219 F F F 23 460-479 136 136 136 230 230 230 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 139 139 139 236 236 236 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 139, Max flush: 236 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.446936914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:06.491295665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.49177908Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.523506527Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.524579784Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.524618466Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.524632603Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52464752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} {"msg":"Crucible b9a17278-479c-412f-ad23-a0b7374cb26a has session id: f5b731d4-bba4-4b82-b9df-4101f35f40ac","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525325003Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Upstairs opts: Upstairs UUID: b9a17278-479c-412f-ad23-a0b7374cb26a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525366844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Crucible stats registered with UUID: b9a17278-479c-412f-ad23-a0b7374cb26a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52538419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:06.525398177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525795491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525820536Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} {"msg":"b9a17278-479c-412f-ad23-a0b7374cb26a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525879043Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525906127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525935701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525950438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525964265Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525977192Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.525990799Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526004246Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526018003Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52603095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52487)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526138287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:61620)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526431223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:57541)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526468495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526930405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.526960089Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.527398074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.5276955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.527732812Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.527990196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528303558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528328252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 235, 235, 230, 236, 236, 210, 235, 220, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528579548Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 138, 136, 139, 139, 127, 138, 132, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528606472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528621679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 235, 235, 230, 236, 236, 210, 235, 220, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528635746Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 138, 136, 139, 139, 127, 138, 132, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528649903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52866365Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 235, 235, 230, 236, 236, 210, 235, 220, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528677017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 138, 136, 139, 139, 127, 138, 132, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528690164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528703431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528724607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528738074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528750511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528762968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"":"downstairs","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528775656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528787203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"0","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52879999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"1","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528812818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"client":"2","":"downstairs","session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528825735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528837532Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"b9a17278-479c-412f-ad23-a0b7374cb26a is now active with session: f5b731d4-bba4-4b82-b9df-4101f35f40ac","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.52884985Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528861487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744,"session_id":"f5b731d4-bba4-4b82-b9df-4101f35f40ac"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:06.528876864Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 8192 data: 14 15 02/30 Write block 67 len 20480 data: 11 11 13 11 10 03/30 Read block 242 len 12288 04/30 Write block 464 len 12288 data: 8 9 10 05/30 Read block 297 len 8192 06/30 Read block 79 len 28672 07/30 Write block 149 len 40960 data: 10 10 9 8 9 11 9 12 13 13 08/30 Read block 521 len 24576 09/30 Read block 252 len 24576 10/30 Write block 119 len 16384 data: 14 15 15 15 11/30 Write block 232 len 16384 data: 13 12 11 13 12/30 Write block 97 len 28672 data: 8 8 6 9 10 10 11 13/30 Read block 215 len 24576 14/30 Read block 274 len 40960 15/30 Read block 202 len 16384 16/30 Write block 416 len 40960 data: 9 10 12 9 11 12 10 11 13 11 17/30 Flush 18/30 Write block 435 len 32768 data: 8 8 8 6 5 4 4 6 19/30 Write block 383 len 12288 data: 10 11 9 20/30 Read block 421 len 32768 21/30 Read block 133 len 32768 22/30 Write block 237 len 16384 data: 12 15 14 14 23/30 Read block 442 len 24576 24/30 Read block 5 len 36864 25/30 Read block 362 len 12288 26/30 Read block 308 len 8192 27/30 Write block 266 len 20480 data: 15 16 17 15 13 28/30 Write block 409 len 28672 data: 14 12 16 14 9 11 11 29/30 Read block 335 len 32768 30/30 Write block 76 len 28672 data: 5 6 7 7 10 9 13 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:06.549729727Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1744} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 140 140 140 237 237 237 T T T 4 080-099 140 140 140 237 237 237 T T T 5 100-119 140 140 140 237 237 237 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 139 139 139 236 236 236 F F F 11 220-239 140 140 140 237 237 237 T T T 12 240-259 136 136 136 228 228 228 T T T 13 260-279 140 140 140 237 237 237 T T T 14 280-299 139 139 139 236 236 236 F F F 15 300-319 138 138 138 234 234 234 F F F 16 320-339 139 139 139 236 236 236 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 136 136 136 227 227 227 F F F 19 380-399 130 130 130 216 216 216 T T T 20 400-419 140 140 140 237 237 237 T T T 21 420-439 140 140 140 237 237 237 T T T 22 440-459 131 131 131 219 219 219 T T T 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 139 139 139 236 236 236 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 140, Max flush: 237 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 141 141 141 238 238 238 F F F 5 100-119 140 140 140 237 237 237 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 139 139 139 236 236 236 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 141 141 141 238 238 238 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 139 139 139 236 236 236 F F F 15 300-319 138 138 138 234 234 234 F F F 16 320-339 139 139 139 236 236 236 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 136 136 136 227 227 227 F F F 19 380-399 141 141 141 238 238 238 F F F 20 400-419 141 141 141 238 238 238 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 141 141 141 238 238 238 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 139 139 139 236 236 236 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 141, Max flush: 238 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.85093252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.895311156Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.896224958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.896281196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.896299562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.896317968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} {"msg":"Crucible a2699dc1-f50c-4d65-bdc2-97d0a2f2a3fd has session id: 87bad865-8065-4357-96e7-c01623b9506a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.89690593Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Upstairs opts: Upstairs UUID: a2699dc1-f50c-4d65-bdc2-97d0a2f2a3fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.896973936Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Crucible stats registered with UUID: a2699dc1-f50c-4d65-bdc2-97d0a2f2a3fd","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897005909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:10.897029054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897438465Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897475417Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} {"msg":"a2699dc1-f50c-4d65-bdc2-97d0a2f2a3fd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897541273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897560439Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897583944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897597801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897611708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897624375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897638242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897651359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897664726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897678663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33769)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897813114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43290)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897853505Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48224)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.897912523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.898674187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.898702731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.898718858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.899242295Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.899285605Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.899395931Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.899850923Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.899992962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 237, 237, 237, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900024675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 141, 141, 140, 140, 140, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900040292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900054339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 237, 237, 237, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900071495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 141, 141, 140, 140, 140, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900085542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900099049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 237, 237, 237, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900112496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 141, 141, 140, 140, 140, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900126123Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.90013995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900152747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:44:10.900165635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.90018441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900197018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"":"downstairs","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900210135Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900221742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"0","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.90023478Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"1","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900247517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"client":"2","":"downstairs","session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900260404Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900274141Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"a2699dc1-f50c-4d65-bdc2-97d0a2f2a3fd is now active with session: 87bad865-8065-4357-96e7-c01623b9506a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900286838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900298666Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752,"session_id":"87bad865-8065-4357-96e7-c01623b9506a"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:10.900310643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 369 len 12288 data: 8 12 12 02/30 Flush 03/30 Write block 448 len 32768 data: 10 9 9 9 9 13 14 12 04/30 Read block 123 len 32768 05/30 Write block 442 len 8192 data: 7 7 06/30 Read block 490 len 32768 07/30 Write block 534 len 12288 data: 15 14 11 08/30 Flush 09/30 Write block 374 len 4096 data: 9 10/30 Read block 476 len 16384 11/30 Read block 363 len 24576 12/30 Read block 84 len 4096 13/30 Flush 14/30 Write block 327 len 40960 data: 7 10 11 11 11 12 10 9 9 10 15/30 Flush 16/30 Write block 118 len 40960 data: 12 15 16 16 16 15 15 14 8 8 17/30 Flush 18/30 Write block 248 len 24576 data: 6 6 6 6 4 6 19/30 Read block 224 len 16384 20/30 Write block 149 len 8192 data: 11 11 21/30 Read block 348 len 40960 22/30 Write block 373 len 16384 data: 11 10 11 10 23/30 Read block 363 len 24576 24/30 Write block 433 len 4096 data: 9 25/30 Read block 127 len 16384 26/30 Read block 506 len 40960 27/30 Write block 258 len 4096 data: 12 28/30 Write block 303 len 8192 data: 11 12 29/30 Read block 520 len 8192 30/30 Read block 28 len 12288 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:10.919800321Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1752} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 141 141 141 238 238 238 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 140 140 140 237 237 237 T T T 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 139 139 139 236 236 236 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 141 141 141 238 238 238 T T T 13 260-279 141 141 141 238 238 238 F F F 14 280-299 139 139 139 236 236 236 F F F 15 300-319 138 138 138 234 234 234 T T T 16 320-339 142 142 142 242 242 242 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 142 142 142 241 241 241 T T T 19 380-399 141 141 141 238 238 238 F F F 20 400-419 141 141 141 238 238 238 F F F 21 420-439 141 141 141 238 238 238 T T T 22 440-459 142 142 142 240 240 240 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 139 139 139 236 236 236 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 142, Max flush: 243 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 138 138 138 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 141 141 141 238 238 238 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 138 138 138 235 235 235 F F F 9 180-199 132 132 132 220 220 220 F F F 10 200-219 139 139 139 236 236 236 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 143 143 143 244 244 244 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 139 139 139 236 236 236 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 142 142 142 242 242 242 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 143 143 143 244 244 244 F F F 19 380-399 141 141 141 238 238 238 F F F 20 400-419 141 141 141 238 238 238 F F F 21 420-439 143 143 143 244 244 244 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 139 139 139 236 236 236 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 143, Max flush: 244 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.164296528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.209981662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.210799085Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.210841456Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.210857172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.210871699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} {"msg":"Crucible b8f07361-bc56-4f56-83f7-fac1b0016026 has session id: 051abcb6-b1b2-4c8d-b471-9652960a6136","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.211427319Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Upstairs opts: Upstairs UUID: b8f07361-bc56-4f56-83f7-fac1b0016026, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.211506152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Crucible stats registered with UUID: b8f07361-bc56-4f56-83f7-fac1b0016026","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.211534535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:15.211549362Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.211913523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.211942987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} {"msg":"b8f07361-bc56-4f56-83f7-fac1b0016026 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212016381Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212033278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212055623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.21206995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212087926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212101253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212270676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212305408Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.21239103Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212474592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"ds_connection connected from Ok(127.0.0.1:51909)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212499506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"ds_connection connected from Ok(127.0.0.1:36894)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212520102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"ds_connection connected from Ok(127.0.0.1:55553)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.212534329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213028542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213232078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213276488Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213699586Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213805044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.213828269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214312154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214419801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 243, 243, 244, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214506032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 141, 141, 142, 142, 143, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214534896Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214551762Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 243, 243, 244, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214566109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 141, 141, 142, 142, 143, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214580316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214594003Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 235, 238, 238, 243, 243, 244, 235, 220, 236, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.21460729Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 141, 141, 142, 142, 143, 138, 132, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214621527Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214635314Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:44:15.21465543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214686063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214705109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214718926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"":"downstairs","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214741831Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214760707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"0","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214775004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"1","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214788241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"client":"2","":"downstairs","session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214801248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214819654Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"b8f07361-bc56-4f56-83f7-fac1b0016026 is now active with session: 051abcb6-b1b2-4c8d-b471-9652960a6136","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214841369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214855046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760,"session_id":"051abcb6-b1b2-4c8d-b471-9652960a6136"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:15.214867093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 16384 data: 13 13 11 12 02/30 Flush 03/30 Write block 250 len 16384 data: 7 7 5 7 04/30 Flush 05/30 Write block 334 len 24576 data: 10 10 11 9 12 14 06/30 Flush 07/30 Write block 52 len 20480 data: 9 10 8 8 8 08/30 Read block 160 len 8192 09/30 Write block 529 len 16384 data: 14 12 15 15 10/30 Read block 478 len 4096 11/30 Write block 546 len 36864 data: 17 13 14 10 9 8 8 9 11 12/30 Write block 178 len 32768 data: 12 11 11 11 14 15 15 15 13/30 Read block 595 len 20480 14/30 Write block 392 len 32768 data: 10 9 13 11 10 10 14 14 15/30 Read block 203 len 24576 16/30 Write block 298 len 28672 data: 10 8 11 12 12 12 13 17/30 Read block 108 len 24576 18/30 Read block 537 len 16384 19/30 Read block 577 len 36864 20/30 Read block 466 len 20480 21/30 Flush 22/30 Write block 239 len 28672 data: 15 15 14 17 14 12 10 23/30 Read block 482 len 36864 24/30 Write block 330 len 16384 data: 12 12 13 11 25/30 Read block 591 len 8192 26/30 Read block 175 len 24576 27/30 Write block 41 len 4096 data: 10 28/30 Write block 236 len 28672 data: 11 13 16 16 16 15 18 29/30 Write block 215 len 32768 data: 5 7 7 8 10 8 8 7 30/30 Write block 292 len 28672 data: 12 11 15 14 15 11 11 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:15.23510023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1760} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 144 144 144 248 248 248 T T T 3 060-079 141 141 141 238 238 238 F F F 4 080-099 141 141 141 238 238 238 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 144 144 144 248 248 248 F F F 10 200-219 139 139 139 236 236 236 T T T 11 220-239 141 141 141 238 238 238 T T T 12 240-259 144 144 144 246 246 246 T T T 13 260-279 141 141 141 238 238 238 F F F 14 280-299 144 144 144 248 248 248 T T T 15 300-319 144 144 144 248 248 248 F F F 16 320-339 144 144 144 247 247 247 T T T 17 340-359 139 139 139 236 236 236 F F F 18 360-379 143 143 143 244 244 244 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 144 144 144 245 245 245 F F F 21 420-439 143 143 143 244 244 244 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 144 144 144 248 248 248 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 144, Max flush: 248 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 145 145 145 249 249 249 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 141 141 141 238 238 238 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 144 144 144 248 248 248 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 145 145 145 249 249 249 F F F 15 300-319 144 144 144 248 248 248 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 139 139 139 236 236 236 F F F 18 360-379 143 143 143 244 244 244 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 144 144 144 245 245 245 F F F 21 420-439 143 143 143 244 244 244 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 144 144 144 248 248 248 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 137 137 137 232 232 232 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 145, Max flush: 249 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.482206873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:19.524861753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.525588335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.557317672Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558282783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558325244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558355627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558376473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} {"msg":"Crucible 70ad1329-b8ad-460c-88de-c8825263842c has session id: f105b3e8-4bc4-4fac-abbc-c1294188d0fd","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558904078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Upstairs opts: Upstairs UUID: 70ad1329-b8ad-460c-88de-c8825263842c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558950268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Crucible stats registered with UUID: 70ad1329-b8ad-460c-88de-c8825263842c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.558979232Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:19.559000937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559419986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559453629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} {"msg":"70ad1329-b8ad-460c-88de-c8825263842c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559511676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559537461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559561126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559574983Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.55958916Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559608266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.55963387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559660015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559686169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559715372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:39221)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559743986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58237)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.559996811Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:53037)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.560021176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.560651969Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.560684702Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.560768984Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.561316176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.561346219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.561419423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.561912096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.561948399Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 249, 238, 238, 243, 243, 244, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562012195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[0]R generation[0..12]: [136, 136, 145, 141, 141, 142, 142, 143, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562043908Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562060004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 249, 238, 238, 243, 243, 244, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562073801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[1]R generation[0..12]: [136, 136, 145, 141, 141, 142, 142, 143, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562087209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562105395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 249, 238, 238, 243, 243, 244, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562119651Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[2]R generation[0..12]: [136, 136, 145, 141, 141, 142, 142, 143, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562133019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562146376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562159053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX"Wait for a query_work_queue command to finish before sending IO ,"pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562178229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562199764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562221809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"":"downstairs","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562235737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562247564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"0","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562260541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"1","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562282496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"client":"2","":"downstairs","session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562302342Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562315239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"70ad1329-b8ad-460c-88de-c8825263842c is now active with session: f105b3e8-4bc4-4fac-abbc-c1294188d0fd","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562327606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562355751Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768,"session_id":"f105b3e8-4bc4-4fac-abbc-c1294188d0fd"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:19.562379015Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 347 len 20480 data: 15 11 13 13 12 02/30 Flush 03/30 Write block 310 len 20480 data: 10 10 10 11 9 04/30 Write block 531 len 12288 data: 16 16 16 05/30 Read block 264 len 36864 06/30 Read block 21 len 36864 07/30 Write block 153 len 24576 data: 10 12 10 13 14 14 08/30 Read block 513 len 20480 09/30 Write block 353 len 4096 data: 13 10/30 Read block 388 len 8192 11/30 Flush 12/30 Write block 407 len 4096 data: 10 13/30 Read block 185 len 40960 14/30 Read block 545 len 36864 15/30 Write block 20 len 16384 data: 9 9 9 10 16/30 Write block 306 len 24576 data: 10 10 10 9 11 11 17/30 Write block 92 len 40960 data: 9 10 8 8 6 9 9 7 10 11 18/30 Read block 423 len 16384 19/30 Flush 20/30 Write block 425 len 24576 data: 12 13 15 12 10 11 21/30 Read block 19 len 36864 22/30 Write block 428 len 16384 data: 13 11 12 9 23/30 Read block 285 len 16384 24/30 Write block 103 len 8192 data: 12 10 25/30 Read block 181 len 4096 26/30 Flush 27/30 Write block 61 len 4096 data: 14 28/30 Write block 444 len 4096 data: 9 29/30 Write block 564 len 32768 data: 10 13 14 13 11 12 9 8 30/30 Write block 145 len 8192 data: 10 13 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:19.579907458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1768} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 145 145 145 249 249 249 F F F 3 060-079 141 141 141 238 238 238 T T T 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 146 146 146 251 251 251 T T T 8 160-179 144 144 144 248 248 248 F F F 9 180-199 144 144 144 248 248 248 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 145 145 145 249 249 249 F F F 15 300-319 146 146 146 252 252 252 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 146 146 146 251 251 251 F F F 18 360-379 143 143 143 244 244 244 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 146 146 146 252 252 252 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 142 142 142 240 240 240 T T T 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 146 146 146 251 251 251 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 137 137 137 232 232 232 T T T 29 580-599 136 136 136 230 230 230 F F F Max gen: 146, Max flush: 253 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 145 145 145 249 249 249 F F F 3 060-079 147 147 147 254 254 254 F F F 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 147 147 147 254 254 254 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 144 144 144 248 248 248 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 145 145 145 249 249 249 F F F 15 300-319 146 146 146 252 252 252 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 146 146 146 251 251 251 F F F 18 360-379 143 143 143 244 244 244 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 146 146 146 252 252 252 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 147 147 147 254 254 254 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 139 139 139 236 236 236 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 146 146 146 251 251 251 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 147 147 147 254 254 254 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 147, Max flush: 254 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.850180552Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.893290394Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.893982584Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.894023835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.894043281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.894058177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} {"msg":"Crucible 1b0c6f49-1fd3-4ca2-b782-638ae15e0aa6 has session id: fbd18c19-3a60-43ac-a37e-7c22f972c9d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.894550121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Upstairs opts: Upstairs UUID: 1b0c6f49-1fd3-4ca2-b782-638ae15e0aa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.894615817Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Crucible stats registered with UUID: 1b0c6f49-1fd3-4ca2-b782-638ae15e0aa6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89464467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:23.894669115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895150411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895179395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} {"msg":"1b0c6f49-1fd3-4ca2-b782-638ae15e0aa6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895238082Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895264646Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895300018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895328062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895357906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89538324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895410685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89543292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895457424Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895479199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57309)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895503904Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61224)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895528989Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62617)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.895703011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.896250403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.896287555Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.896968047Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.897053269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.897635543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.897659477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.898133335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.898624558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[0]R flush_numbers[0..12]: [230, 252, 249, 254, 252, 253, 243, 254, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899101675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[0]R generation[0..12]: [136, 146, 145, 147, 146, 146, 142, 147, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899135167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899153854Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[1]R flush_numbers[0..12]: [230, 252, 249, 254, 252, 253, 243, 254, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89917124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[1]R generation[0..12]: [136, 146, 145, 147, 146, 146, 142, 147, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899188096Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-23T17:44:23.899205023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[2]R flush_numbers[0..12]: [230, 252, 249, 254, 252, 253, 243, 254, 248, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899228118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[2]R generation[0..12]: [136, 146, 145, 147, 146, 146, 142, 147, 144, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899244674Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89926133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899276907Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899292813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89930811Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899323697Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"":"downstairs","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899339423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.89935375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"0","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899372006Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 client":"1","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899392562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"client":"2","":"downstairs","session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899408898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899423695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"1b0c6f49-1fd3-4ca2-b782-638ae15e0aa6 is now active with session: fbd18c19-3a60-43ac-a37e-7c22f972c9d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899439432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899453858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776,"session_id":"fbd18c19-3a60-43ac-a37e-7c22f972c9d6"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:23.899468635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 32768 data: 13 13 13 14 11 11 11 11 02/30 Read block 111 len 28672 03/30 Read block 51 len 20480 04/30 Flush 05/30 Write block 197 len 16384 data: 8 6 8 9 06/30 Read block 107 len 12288 07/30 Write block 272 len 12288 data: 14 17 15 08/30 Write block 157 len 8192 data: 15 15 09/30 Read block 314 len 28672 10/30 Read block 426 len 16384 11/30 Write block 400 len 20480 data: 12 13 12 12 10 12/30 Read block 437 len 16384 13/30 Read block 311 len 32768 14/30 Read block 109 len 16384 15/30 Write block 211 len 28672 data: 12 10 6 6 6 8 8 16/30 Write block 404 len 20480 data: 11 8 8 11 12 17/30 Flush 18/30 Write block 474 len 36864 data: 11 10 11 11 11 10 13 13 13 19/30 Write block 190 len 28672 data: 14 12 16 12 10 11 7 20/30 Write block 189 len 16384 data: 13 15 13 17 21/30 Flush 22/30 Write block 454 len 36864 data: 15 13 14 12 13 11 11 10 12 23/30 Read block 47 len 36864 24/30 Write block 371 len 28672 data: 13 13 12 11 12 11 8 25/30 Read block 470 len 40960 26/30 Write block 361 len 4096 data: 6 27/30 Write block 44 len 24576 data: 8 10 11 10 11 10 28/30 Write block 281 len 36864 data: 20 19 16 14 15 16 16 14 13 29/30 Read block 468 len 40960 30/30 Write block 354 len 24576 data: 11 10 10 9 9 9 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:23.919671569Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1776} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 145 145 145 249 249 249 T T T 3 060-079 147 147 147 254 254 254 F F F 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 148 148 148 256 256 256 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 148 148 148 257 257 257 F F F 10 200-219 148 148 148 256 256 256 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 148 148 148 256 256 256 F F F 14 280-299 145 145 145 249 249 249 T T T 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 146 146 146 251 251 251 T T T 18 360-379 143 143 143 244 244 244 T T T 19 380-399 144 144 144 248 248 248 F F F 20 400-419 148 148 148 256 256 256 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 147 147 147 254 254 254 T T T 23 460-479 148 148 148 257 257 257 T T T 24 480-499 148 148 148 257 257 257 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 146 146 146 251 251 251 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 147 147 147 254 254 254 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 148, Max flush: 257 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 147 147 147 254 254 254 F F F 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 148 148 148 256 256 256 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 148 148 148 257 257 257 F F F 10 200-219 148 148 148 256 256 256 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 148 148 148 256 256 256 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 149 149 149 258 258 258 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 148 148 148 256 256 256 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 149 149 149 258 258 258 F F F 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 139 139 139 236 236 236 F F F 26 520-539 146 146 146 251 251 251 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 147 147 147 254 254 254 F F F 29 580-599 136 136 136 230 230 230 F F F Max gen: 149, Max flush: 258 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.164459554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.210709395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.211453714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.211497624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.211513871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.211529657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} {"msg":"Crucible 4084fe67-0b0e-48aa-9c54-87b8b900325f has session id: 01f652f4-9fbd-4e1c-97c5-b59f8d84ea90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212138726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Upstairs opts: Upstairs UUID: 4084fe67-0b0e-48aa-9c54-87b8b900325f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212183686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Crucible stats registered with UUID: 4084fe67-0b0e-48aa-9c54-87b8b900325f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212207081Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:28.212227346Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212686007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212753052Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} {"msg":"4084fe67-0b0e-48aa-9c54-87b8b900325f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212770179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212789285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.21281074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212824217Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212837894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212850662Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212864228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212885234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212912068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.212936563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"ds_connection connected from Ok(127.0.0.1:45128)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.213220551Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"ds_connection connected from Ok(127.0.0.1:53211)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.213263372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"ds_connection connected from Ok(127.0.0.1:54496)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.213282998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.213859773Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.213888517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.214245649Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.214587165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.214627397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.214920593Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215227347Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215325995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[0]R flush_numbers[0..12]: [230, 252, 258, 254, 252, 253, 243, 256, 248, 257, 256, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215439511Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[0]R generation[0..12]: [136, 146, 149, 147, 146, 146, 142, 148, 144, 148, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215460036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215476902Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[1]R flush_numbers[0..12]: [230, 252, 258, 254, 252, 253, 243, 256, 248, 257, 256, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215499977Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[1]R generation[0..12]: [136, 146, 149, 147, 146, 146, 142, 148, 144, 148, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215521453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215547407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[2]R flush_numbers[0..12]: [230, 252, 258, 254, 252, 253, 243, 256, 248, 257, 256, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215573132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[2]R generation[0..12]: [136, 146, 149, 147, 146, 146, 142, 148, 144, 148, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215594257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215610813Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215635248Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215648016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215660413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.2156728Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"":"downstairs","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215685367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215696895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"0","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215709892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"1","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215722529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"client":"2","":"downstairs","session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215735486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215747444Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"4084fe67-0b0e-48aa-9c54-87b8b900325f is now active with session: 01f652f4-9fbd-4e1c-97c5-b59f8d84ea90","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215759721Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215771239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784,"session_id":"01f652f4-9fbd-4e1c-97c5-b59f8d84ea90"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:28.215782946Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 72 len 40960 data: 11 13 10 9 6 7 8 8 11 10 02/30 Flush 03/30 Write block 387 len 40960 data: 11 10 9 9 9 11 10 14 12 11 04/30 Write block 569 len 36864 data: 13 10 9 9 9 6 8 7 9 05/30 Write block 202 len 12288 data: 14 13 13 06/30 Read block 521 len 8192 07/30 Write block 519 len 36864 data: 7 9 11 11 11 11 14 14 10 08/30 Read block 320 len 8192 09/30 Write block 151 len 12288 data: 10 9 11 10/30 Write block 207 len 24576 data: 15 15 11 12 13 11 11/30 Write block 93 len 8192 data: 11 9 12/30 Write block 81 len 36864 data: 11 14 13 14 21 18 17 16 16 13/30 Read block 74 len 36864 14/30 Write block 135 len 4096 data: 12 15/30 Read block 50 len 24576 16/30 Read block 149 len 12288 17/30 Write block 125 len 4096 data: 15 18/30 Write block 189 len 16384 data: 14 16 14 18 19/30 Flush 20/30 Write block 380 len 12288 data: 10 8 9 21/30 Read block 227 len 4096 22/30 Read block 198 len 28672 23/30 Flush 24/30 Write block 452 len 16384 data: 10 14 16 14 25/30 Read block 14 len 12288 26/30 Flush 27/30 Write block 210 len 40960 data: 13 14 12 7 7 7 9 9 9 11 28/30 Write block 266 len 12288 data: 16 17 18 29/30 Write block 577 len 36864 data: 10 9 10 11 11 10 7 9 7 30/30 Read block 384 len 24576 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:28.235068309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1784} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 150 150 150 259 259 259 F F F 4 080-099 150 150 150 260 260 260 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 150 150 150 260 260 260 F F F 10 200-219 150 150 150 260 260 260 T T T 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 148 148 148 256 256 256 T T T 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 149 149 149 258 258 258 F F F 19 380-399 150 150 150 261 261 261 F F F 20 400-419 148 148 148 256 256 256 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 150 150 150 262 262 262 F F F 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 150 150 150 260 260 260 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 150 150 150 260 260 260 T T T 29 580-599 136 136 136 230 230 230 T T T Max gen: 150, Max flush: 262 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 150 150 150 259 259 259 F F F 4 080-099 150 150 150 260 260 260 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 150 150 150 260 260 260 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 145 145 145 249 249 249 F F F 13 260-279 151 151 151 263 263 263 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 149 149 149 258 258 258 F F F 19 380-399 150 150 150 261 261 261 F F F 20 400-419 148 148 148 256 256 256 F F F 21 420-439 146 146 146 253 253 253 F F F 22 440-459 150 150 150 262 262 262 F F F 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 150 150 150 260 260 260 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 144 144 144 248 248 248 F F F 28 560-579 151 151 151 263 263 263 F F F 29 580-599 151 151 151 263 263 263 F F F Max gen: 151, Max flush: 263 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.473577453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.518098369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.518861463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.518897786Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.518911412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.518925349Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} {"msg":"Crucible 78f4793f-56bc-4316-b9b0-39dd594c2879 has session id: a742397e-063d-418e-84f6-5822502571d4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.519548994Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Upstairs opts: Upstairs UUID: 78f4793f-56bc-4316-b9b0-39dd594c2879, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.519595374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Crucible stats registered with UUID: 78f4793f-56bc-4316-b9b0-39dd594c2879","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.519618529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:32.519632426Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520037349Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520062953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} {"msg":"78f4793f-56bc-4316-b9b0-39dd594c2879 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520134038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520151014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520173769Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520188146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520201843Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.52021464Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520228387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520241124Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520265289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520289984Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51690)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520346002Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42222)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520383944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36114)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.520419106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.521066596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.52109541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.52169267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.521796598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.522255268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.52238316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.522881982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523376415Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[0]R flush_numbers[0..12]: [230, 252, 258, 259, 260, 253, 260, 260, 248, 260, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523889284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[0]R generation[0..12]: [136, 146, 149, 150, 150, 146, 150, 150, 144, 150, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523914759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523934284Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[1]R flush_numbers[0..12]: [230, 252, 258, 259, 260, 253, 260, 260, 248, 260, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523951541Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[1]R generation[0..12]: [136, 146, 149, 150, 150, 146, 150, 150, 144, 150, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523973436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.523991022Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[2]R flush_numbers[0..12]: [230, 252, 258, 259, 260, 253, 260, 260, 248, 260, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524021925Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [136, 146, 149, 150, 150, 146, 150, 150, 144, 150, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524048999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524079723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524109606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.52413826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524166884Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524195308Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"":"downstairs","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524225641Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524250076Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"0","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524267192Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"1","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524282819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"client":"2","":"downstairs","session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524298716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524313622Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"78f4793f-56bc-4316-b9b0-39dd594c2879 is now active with session: a742397e-063d-418e-84f6-5822502571d4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524333118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524347885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792,"session_id":"a742397e-063d-418e-84f6-5822502571d4"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:32.524362682Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 61 len 4096 data: 15 02/30 Read block 37 len 12288 03/30 Read block 133 len 36864 04/30 Write block 180 len 32768 data: 12 12 15 16 16 16 16 16 05/30 Flush 06/30 Write block 71 len 12288 data: 11 12 14 07/30 Write block 509 len 12288 data: 10 10 9 08/30 Write block 546 len 16384 data: 18 14 15 11 09/30 Read block 449 len 8192 10/30 Read block 293 len 28672 11/30 Write block 192 len 16384 data: 19 13 11 12 12/30 Write block 241 len 40960 data: 16 19 15 13 11 8 7 7 7 8 13/30 Read block 173 len 8192 14/30 Read block 457 len 32768 15/30 Write block 367 len 36864 data: 8 9 9 13 14 14 13 12 13 16/30 Read block 567 len 24576 17/30 Read block 89 len 24576 18/30 Read block 435 len 8192 19/30 Read block 20 len 36864 20/30 Write block 410 len 20480 data: 13 17 15 10 12 21/30 Write block 36 len 32768 data: 14 12 13 11 10 11 8 6 22/30 Write block 164 len 4096 data: 12 23/30 Read block 114 len 32768 24/30 Read block 104 len 4096 25/30 Read block 275 len 36864 26/30 Flush 27/30 Write block 576 len 40960 data: 8 11 10 11 12 12 11 8 10 8 28/30 Write block 437 len 16384 data: 9 7 6 5 29/30 Write block 268 len 28672 data: 19 16 14 13 15 18 16 30/30 Read block 210 len 12288 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 2 4 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:32.54412755Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1792} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 152 152 152 265 265 265 F F F 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 150 150 150 260 260 260 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 152 152 152 265 265 265 F F F 13 260-279 151 151 151 263 263 263 T T T 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 152 152 152 265 265 265 F F F 19 380-399 150 150 150 261 261 261 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 146 146 146 253 253 253 T T T 22 440-459 150 150 150 262 262 262 T T T 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 152 152 152 265 265 265 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 151 151 151 263 263 263 T T T 29 580-599 151 151 151 263 263 263 T T T Max gen: 152, Max flush: 265 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 230 230 230 F F F 1 020-039 152 152 152 265 265 265 F F F 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 150 150 150 260 260 260 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 152 152 152 265 265 265 F F F 13 260-279 153 153 153 266 266 266 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 152 152 152 265 265 265 F F F 19 380-399 150 150 150 261 261 261 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 153 153 153 266 266 266 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 152 152 152 265 265 265 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 153 153 153 266 266 266 F F F Max gen: 153, Max flush: 266 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.790859183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.834656655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.835494644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.835538764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.83555795Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.835573307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} {"msg":"Crucible a7b32541-df94-4950-bcfa-5a2da9d73791 has session id: 07faa94d-b712-4517-83a7-bc144d9a4d82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836122018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Upstairs opts: Upstairs UUID: a7b32541-df94-4950-bcfa-5a2da9d73791, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836167448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Crucible stats registered with UUID: a7b32541-df94-4950-bcfa-5a2da9d73791","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836193382Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:36.836216407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836562932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836588127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} {"msg":"a7b32541-df94-4950-bcfa-5a2da9d73791 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836655502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836672609Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836695693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.83670999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836725267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836738494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836752871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836770227Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836785074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836836013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"ds_connection connected from Ok(127.0.0.1:52627)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836868706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"ds_connection connected from Ok(127.0.0.1:36114)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.83689537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"ds_connection connected from Ok(127.0.0.1:63909)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.836921944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.837716852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.837745486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.838369491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.838401404Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.839035997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.839070599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.839782585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840279757Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[0]R flush_numbers[0..12]: [230, 265, 265, 265, 260, 253, 260, 260, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840778579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[0]R generation[0..12]: [136, 152, 152, 152, 150, 146, 150, 150, 152, 152, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840800144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840815941Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[1]R flush_numbers[0..12]: [230, 265, 265, 265, 260, 253, 260, 260, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840830048Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[1]R generation[0..12]: [136, 152, 152, 152, 150, 146, 150, 150, 152, 152, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840843945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840857722Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[2]R flush_numbers[0..12]: [230, 265, 265, 265, 260, 253, 260, 260, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840871029Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[2]R generation[0..12]: [136, 152, 152, 152, 150, 146, 150, 150, 152, 152, 151, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840884296Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-01-23T17:44:36.840904112Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840916759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840929427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840941724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840954281Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"":"downstairs","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840966968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840981675Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"0","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.840995032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"1","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841007639Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"client":"2","":"downstairs","session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841020417Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841032084Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"a7b32541-df94-4950-bcfa-5a2da9d73791 is now active with session: 07faa94d-b712-4517-83a7-bc144d9a4d82","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841044531Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841060708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800,"session_id":"07faa94d-b712-4517-83a7-bc144d9a4d82"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:36.841072765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 381 len 8192 data: 9 10 02/30 Write block 436 len 36864 data: 9 10 8 7 6 5 8 8 10 03/30 Read block 416 len 12288 04/30 Read block 320 len 40960 05/30 Read block 589 len 24576 06/30 Read block 549 len 20480 07/30 Write block 3 len 16384 data: 8 9 13 14 08/30 Read block 492 len 36864 09/30 Write block 519 len 32768 data: 8 10 12 12 12 12 15 15 10/30 Read block 511 len 4096 11/30 Write block 94 len 4096 data: 10 12/30 Read block 132 len 24576 13/30 Read block 317 len 16384 14/30 Read block 210 len 20480 15/30 Write block 484 len 16384 data: 16 15 13 12 16/30 Read block 585 len 12288 17/30 Read block 379 len 8192 18/30 Write block 530 len 40960 data: 13 17 17 17 16 15 12 15 14 17 19/30 Read block 301 len 36864 20/30 Read block 536 len 16384 21/30 Flush 22/30 Write block 250 len 12288 data: 9 8 6 23/30 Write block 302 len 12288 data: 14 14 15 24/30 Write block 369 len 16384 data: 10 14 15 15 25/30 Read block 440 len 8192 26/30 Write block 479 len 16384 data: 11 14 14 14 27/30 Read block 164 len 24576 28/30 Write block 226 len 4096 data: 11 29/30 Read block 37 len 32768 30/30 Write block 28 len 8192 data: 13 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 8 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:36.860179876Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1800} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 267 267 267 F F F 1 020-039 152 152 152 265 265 265 T T T 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 145 145 145 249 249 249 T T T 12 240-259 152 152 152 265 265 265 T T T 13 260-279 153 153 153 266 266 266 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 148 148 148 255 255 255 T T T 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 152 152 152 265 265 265 T T T 19 380-399 154 154 154 267 267 267 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 149 149 149 258 258 258 T T T 24 480-499 154 154 154 267 267 267 T T T 25 500-519 154 154 154 267 267 267 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 153 153 153 266 266 266 F F F Max gen: 154, Max flush: 267 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 267 267 267 F F F 1 020-039 155 155 155 268 268 268 F F F 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 146 146 146 253 253 253 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 153 153 153 266 266 266 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 155 155 155 268 268 268 F F F 16 320-339 145 145 145 249 249 249 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 155 155 155 268 268 268 F F F 19 380-399 154 154 154 267 267 267 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 155 155 155 268 268 268 F F F 24 480-499 155 155 155 268 268 268 F F F 25 500-519 154 154 154 267 267 267 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 153 153 153 266 266 266 F F F Max gen: 155, Max flush: 268 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.122456912Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.166652018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.167459623Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.167511422Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.167531268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.167550823Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} {"msg":"Crucible 598f40df-5e32-44a3-86a2-1d02061713b4 has session id: ce655f28-a5bc-4dcc-9724-e507bcd52c7c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168176118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Upstairs opts: Upstairs UUID: 598f40df-5e32-44a3-86a2-1d02061713b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168221868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Crucible stats registered with UUID: 598f40df-5e32-44a3-86a2-1d02061713b4","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168247723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:41.168269238Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168639158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168662953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} {"msg":"598f40df-5e32-44a3-86a2-1d02061713b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168730778Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168752403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168798753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16881363Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168836455Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16885833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168882405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16890335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168927335Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16895244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48967)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.168975755Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60169)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16899915Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:41.169025964Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:41.169052138Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:41.169233739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.169356892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.169638941Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.16978315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.170183533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.170323113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.170760178Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:41.170848629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.170753874Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64229)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.170890225Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.171913333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.172514223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[0]R flush_numbers[0..12]: [267, 268, 265, 265, 267, 253, 260, 260, 265, 265, 263, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173156323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[0]R generation[0..12]: [154, 155, 152, 152, 154, 146, 150, 150, 152, 152, 151, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173186157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173205563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[1]R flush_numbers[0..12]: [267, 268, 265, 265, 267, 253, 260, 260, 265, 265, 263, 268]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173228768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[1]R generation[0..12]: [154, 155, 152, 152, 154, 146, 150, 150, 152, 152, 151, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173247074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.1732651Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[2]R flush_numbers[0..12]: [267, 268, 265, 265, 267, 253, 260, 260, 265, 265, 263, 268]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-23T17:44:51.173284436Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[2]R generation[0..12]: [154, 155, 152, 152, 154, 146, 150, 150, 152, 152, 151, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173321058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173354621Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173386084Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173418177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.17345118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173483643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"":"downstairs","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173514316Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.17354255Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"0","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173565725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"1","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:44:51.173583461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"client":"2","":"downstairs","session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173601437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173620443Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"598f40df-5e32-44a3-86a2-1d02061713b4 is now active with session: ce655f28-a5bc-4dcc-9724-e507bcd52c7c","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.17363691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173651996Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808,"session_id":"ce655f28-a5bc-4dcc-9724-e507bcd52c7c"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:51.173667343Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 28672 data: 8 10 10 10 11 10 9 02/30 Read block 337 len 16384 03/30 Write block 260 len 32768 data: 12 10 11 11 13 15 17 18 04/30 Flush 05/30 Write block 510 len 36864 data: 11 10 8 8 10 8 8 9 6 06/30 Write block 343 len 8192 data: 16 14 07/30 Write block 24 len 40960 data: 11 10 10 12 14 13 11 11 13 12 08/30 Flush 09/30 Write block 581 len 20480 data: 13 12 9 11 9 10/30 Read block 571 len 28672 11/30 Write block 235 len 32768 data: 14 12 14 17 17 17 17 20 12/30 Read block 567 len 28672 13/30 Read block 124 len 36864 14/30 Write block 247 len 36864 data: 8 8 8 10 9 7 8 6 8 15/30 Read block 340 len 20480 16/30 Read block 544 len 24576 17/30 Write block 330 len 28672 data: 13 13 14 12 11 11 12 18/30 Write block 295 len 36864 data: 15 16 12 12 9 12 14 15 15 19/30 Read block 452 len 4096 20/30 Write block 462 len 12288 data: 13 8 9 21/30 Read block 82 len 16384 22/30 Write block 270 len 24576 data: 15 14 16 19 17 13 23/30 Read block 305 len 20480 24/30 Write block 338 len 4096 data: 13 25/30 Read block 255 len 40960 26/30 Write block 301 len 20480 data: 15 16 16 16 12 27/30 Read block 48 len 20480 28/30 Write block 110 len 32768 data: 12 14 13 11 10 12 12 9 29/30 Write block 308 len 20480 data: 12 10 12 12 11 30/30 Read block 248 len 36864 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 21 22 22 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:51.195897747Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1808} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 269 269 269 F F F 1 020-039 156 156 156 270 270 270 F F F 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 146 146 146 253 253 253 T T T 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 155 155 155 268 268 268 T T T 12 240-259 155 155 155 268 268 268 T T T 13 260-279 156 156 156 269 269 269 T T T 14 280-299 149 149 149 258 258 258 T T T 15 300-319 155 155 155 268 268 268 T T T 16 320-339 145 145 145 249 249 249 T T T 17 340-359 156 156 156 270 270 270 F F F 18 360-379 155 155 155 268 268 268 F F F 19 380-399 154 154 154 267 267 267 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 155 155 155 268 268 268 T T T 24 480-499 155 155 155 268 268 268 F F F 25 500-519 156 156 156 270 270 270 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 153 153 153 266 266 266 T T T Max gen: 156, Max flush: 270 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 269 269 269 F F F 1 020-039 156 156 156 270 270 270 F F F 2 040-059 152 152 152 265 265 265 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 157 157 157 271 271 271 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 152 152 152 265 265 265 F F F 9 180-199 152 152 152 265 265 265 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 157 157 157 271 271 271 F F F 12 240-259 157 157 157 271 271 271 F F F 13 260-279 157 157 157 271 271 271 F F F 14 280-299 157 157 157 271 271 271 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 155 155 155 268 268 268 F F F 19 380-399 154 154 154 267 267 267 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 157 157 157 271 271 271 F F F 24 480-499 155 155 155 268 268 268 F F F 25 500-519 156 156 156 270 270 270 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 157 157 157 271 271 271 F F F Max gen: 157, Max flush: 271 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.464180801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.507193144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.507919926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.507963707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.507980183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.50799525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} {"msg":"Crucible 19deee9f-c687-4d39-901c-343652f0b37a has session id: 7b7fd217-e430-49a5-99a5-e329ce972eac","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.508528974Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Upstairs opts: Upstairs UUID: 19deee9f-c687-4d39-901c-343652f0b37a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.508580213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Crucible stats registered with UUID: 19deee9f-c687-4d39-901c-343652f0b37a","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.508605978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:55.508629303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509064798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509095691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} {"msg":"19deee9f-c687-4d39-901c-343652f0b37a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509168536Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.5091944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509225163Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.50924034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509254437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509267114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509281951Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509294688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509307996Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509321213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52795)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509448955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:49058)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509494205Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:37695)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.509524239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.510177757Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.510216589Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.510845183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.510876796Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.511522056Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.51154704Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.511563577Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512077186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[0]R flush_numbers[0..12]: [269, 270, 265, 265, 267, 271, 260, 260, 265, 265, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512556892Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[0]R generation[0..12]: [156, 156, 152, 152, 154, 157, 150, 150, 152, 152, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512579227Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512594184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[1]R flush_numbers[0..12]: [269, 270, 265, 265, 267, 271, 260, 260, 265, 265, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512607761Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[1]R generation[0..12]: [156, 156, 152, 152, 154, 157, 150, 150, 152, 152, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512621358Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512635045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[2]R flush_numbers[0..12]: [269, 270, 265, 265, 267, 271, 260, 260, 265, 265, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512648132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[2]R generation[0..12]: [156, 156, 152, 152, 154, 157, 150, 150, 152, 152, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512661109Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512674266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512690593Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.51270385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512717367Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512739102Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"":"downstairs","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512752579Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512765536Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"0","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512778714Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"1","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512791451Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"client":"2","":"downstairs","session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512804328Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512816285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"19deee9f-c687-4d39-901c-343652f0b37a is now active with session: 7b7fd217-e430-49a5-99a5-e329ce972eac","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512828643Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.51284007Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816,"session_id":"7b7fd217-e430-49a5-99a5-e329ce972eac"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:55.512851898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 36864 data: 10 12 11 15 13 12 11 15 15 02/30 Read block 11 len 20480 03/30 Read block 176 len 12288 04/30 Write block 540 len 28672 data: 19 21 22 19 19 16 19 05/30 Write block 177 len 16384 data: 8 13 12 13 06/30 Read block 218 len 20480 07/30 Write block 270 len 32768 data: 16 15 17 20 18 14 12 13 08/30 Read block 112 len 28672 09/30 Read block 566 len 8192 10/30 Write block 484 len 28672 data: 17 16 14 13 13 14 12 11/30 Write block 584 len 32768 data: 12 10 9 6 8 7 6 7 12/30 Read block 255 len 4096 13/30 Read block 2 len 8192 14/30 Read block 159 len 4096 15/30 Flush 16/30 Write block 473 len 4096 data: 10 17/30 Write block 391 len 28672 data: 11 13 12 16 14 13 12 18/30 Write block 251 len 20480 data: 10 8 9 7 9 19/30 Flush 20/30 Write block 499 len 16384 data: 9 10 10 13 21/30 Write block 40 len 24576 data: 11 12 9 7 9 11 22/30 Flush 23/30 Write block 281 len 4096 data: 21 24/30 Read block 255 len 28672 25/30 Read block 521 len 32768 26/30 Read block 434 len 8192 27/30 Write block 483 len 20480 data: 15 18 17 15 14 28/30 Write block 32 len 16384 data: 14 13 15 13 29/30 Write block 325 len 24576 data: 10 8 8 11 12 14 30/30 Read block 142 len 36864 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:55.532710356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1816} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 269 269 269 F F F 1 020-039 156 156 156 270 270 270 T T T 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 157 157 157 271 271 271 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 158 158 158 272 272 272 F F F 9 180-199 158 158 158 272 272 272 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 157 157 157 271 271 271 F F F 12 240-259 158 158 158 273 273 273 F F F 13 260-279 158 158 158 272 272 272 F F F 14 280-299 157 157 157 271 271 271 T T T 15 300-319 157 157 157 271 271 271 F F F 16 320-339 157 157 157 271 271 271 T T T 17 340-359 156 156 156 270 270 270 F F F 18 360-379 155 155 155 268 268 268 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 158 158 158 273 273 273 F F F 24 480-499 158 158 158 274 274 274 T T T 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 158 158 158 272 272 272 F F F Max gen: 158, Max flush: 274 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 269 269 269 F F F 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 157 157 157 271 271 271 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 158 158 158 272 272 272 F F F 9 180-199 158 158 158 272 272 272 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 157 157 157 271 271 271 F F F 12 240-259 158 158 158 273 273 273 F F F 13 260-279 158 158 158 272 272 272 F F F 14 280-299 159 159 159 275 275 275 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 159 159 159 275 275 275 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 155 155 155 268 268 268 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 154 154 154 267 267 267 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 158 158 158 273 273 273 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 158 158 158 272 272 272 F F F Max gen: 159, Max flush: 275 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.768483953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:59.811765886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.812271916Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.843335547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.844166367Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.844205789Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.844226354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.844241521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"Crucible cfd831b0-2779-4ad2-85e3-8df5f3b92b02 has session id: 57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845011395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Upstairs opts: Upstairs UUID: cfd831b0-2779-4ad2-85e3-8df5f3b92b02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845047496Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Crucible stats registered with UUID: cfd831b0-2779-4ad2-85e3-8df5f3b92b02","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845062693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:59.845075501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845470695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.84549472Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"cfd831b0-2779-4ad2-85e3-8df5f3b92b02 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845562615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845579631Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845603026Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845616683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.84563061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845646177Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845671631Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845696676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.8457246Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845756413Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:59886)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845782677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:49549)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845821409Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:43786)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.845847183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.846548841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.846578245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.846593272Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847216147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.84724866Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847265156Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847850859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847876894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[0]R flush_numbers[0..12]: [269, 275, 274, 265, 267, 271, 260, 260, 272, 272, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.84793918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[0]R generation[0..12]: [156, 159, 158, 152, 154, 157, 150, 150, 158, 158, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847959756Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847978582Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[1]R flush_numbers[0..12]: [269, 275, 274, 265, 267, 271, 260, 260, 272, 272, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.847992369Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[1]R generation[0..12]: [156, 159, 158, 152, 154, 157, 150, 150, 158, 158, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848006086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848020543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[2]R flush_numbers[0..12]: [269, 275, 274, 265, 267, 271, 260, 260, 272, 272, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.84803412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[2]R generation[0..12]: [156, 159, 158, 152, 154, 157, 150, 150, 158, 158, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848048246Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848061854Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848074221Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:44:59.848086858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848103885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848116172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"":"downstairs","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848129019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848140626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"0","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848153604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"1","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848166191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"client":"2","":"downstairs","session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848183027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848195405Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"cfd831b0-2779-4ad2-85e3-8df5f3b92b02 is now active with session: 57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848208062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848219809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:44:59.848230287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824,"session_id":"57a1ae25-a818-4ccc-8ecb-ee7c4cbb62fe"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 476 len 4096 data: 12 02/30 Write block 410 len 8192 data: 14 18 03/30 Write block 574 len 16384 data: 7 9 9 12 04/30 Read block 13 len 8192 05/30 Read block 245 len 16384 06/30 Write block 433 len 28672 data: 10 8 9 10 11 9 8 07/30 Write block 360 len 8192 data: 8 7 08/30 Read block 145 len 16384 09/30 Read block 294 len 12288 10/30 Flush 11/30 Write block 424 len 40960 data: 14 13 14 16 14 12 13 10 10 11 12/30 Read block 584 len 16384 13/30 Write block 102 len 24576 data: 11 13 11 11 12 11 14/30 Read block 313 len 24576 15/30 Read block 61 len 16384 16/30 Read block 454 len 12288 17/30 Write block 584 len 20480 data: 13 11 10 7 9 18/30 Read block 336 len 8192 19/30 Flush 20/30 Write block 412 len 16384 data: 16 11 13 12 21/30 Flush 22/30 Write block 190 len 16384 data: 17 15 20 14 23/30 Read block 260 len 32768 24/30 Write block 10 len 36864 data: 15 15 16 15 14 12 12 9 9 25/30 Read block 583 len 4096 26/30 Read block 513 len 20480 27/30 Read block 87 len 36864 28/30 Write block 361 len 8192 data: 8 6 29/30 Write block 245 len 40960 data: 12 9 9 9 9 11 11 9 10 8 30/30 Write block 425 len 36864 data: 14 15 17 15 13 14 11 11 12 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:44:59.867857995Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1824} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 269 269 269 T T T 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 160 160 160 277 277 277 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 158 158 158 272 272 272 F F F 9 180-199 158 158 158 272 272 272 T T T 10 200-219 151 151 151 263 263 263 F F F 11 220-239 157 157 157 271 271 271 F F F 12 240-259 158 158 158 273 273 273 T T T 13 260-279 158 158 158 272 272 272 F F F 14 280-299 159 159 159 275 275 275 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 159 159 159 275 275 275 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 160 160 160 276 276 276 T T T 19 380-399 158 158 158 273 273 273 F F F 20 400-419 160 160 160 278 278 278 F F F 21 420-439 160 160 160 277 277 277 T T T 22 440-459 154 154 154 267 267 267 F F F 23 460-479 160 160 160 276 276 276 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 160, Max flush: 278 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 279 279 279 F F F 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 160 160 160 277 277 277 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 158 158 158 272 272 272 F F F 9 180-199 161 161 161 279 279 279 F F F 10 200-219 151 151 151 263 263 263 F F F 11 220-239 157 157 157 271 271 271 F F F 12 240-259 161 161 161 279 279 279 F F F 13 260-279 158 158 158 272 272 272 F F F 14 280-299 159 159 159 275 275 275 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 159 159 159 275 275 275 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 161 161 161 279 279 279 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 160 160 160 278 278 278 F F F 21 420-439 161 161 161 279 279 279 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 160 160 160 276 276 276 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 161, Max flush: 279 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.145820404Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:04.189516738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.189939517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.221154265Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.221779459Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.221815552Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.221830498Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.221845615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} {"msg":"Crucible fc84e28e-c1ad-41ef-9eaa-87cb81d777b3 has session id: 5b237a7c-7d3a-4d7d-8c79-104f501b0bfe","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.222496384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Upstairs opts: Upstairs UUID: fc84e28e-c1ad-41ef-9eaa-87cb81d777b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.222532926Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Crucible stats registered with UUID: fc84e28e-c1ad-41ef-9eaa-87cb81d777b3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.222548213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:04.22256091Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.222949986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.22297421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} {"msg":"fc84e28e-c1ad-41ef-9eaa-87cb81d777b3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223026809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223053233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223081377Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223097444Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223111991Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223125188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223138755Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223151192Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223164159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223226136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39896)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223246092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:35675)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223490219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:49604)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.223524061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.224004667Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.224043679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.224195126Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.22459321Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.224681521Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.224702986Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225170665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225275152Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[0]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 277, 260, 260, 272, 279, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225344967Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[0]R generation[0..12]: [161, 159, 158, 152, 154, 160, 150, 150, 158, 161, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225363753Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.22537851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[1]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 277, 260, 260, 272, 279, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225392307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[1]R generation[0..12]: [161, 159, 158, 152, 154, 160, 150, 150, 158, 161, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225405734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225419471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[2]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 277, 260, 260, 272, 279, 263, 271]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225432858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[2]R generation[0..12]: [161, 159, 158, 152, 154, 160, 150, 150, 158, 161, 151, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225446065Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225459652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225476679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:45:04.225490285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225507392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225519789Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"":"downstairs","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225532596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225544264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"0","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225557291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"1","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225570038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"client":"2","":"downstairs","session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225582865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225594663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"fc84e28e-c1ad-41ef-9eaa-87cb81d777b3 is now active with session: 5b237a7c-7d3a-4d7d-8c79-104f501b0bfe","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.22560709Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225618888Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832,"session_id":"5b237a7c-7d3a-4d7d-8c79-104f501b0bfe"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:04.225630655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 8192 data: 12 13 02/30 Write block 272 len 8192 data: 18 21 03/30 Write block 114 len 24576 data: 11 13 13 10 13 16 04/30 Read block 5 len 4096 05/30 Write block 291 len 12288 data: 14 13 12 06/30 Read block 249 len 24576 07/30 Read block 129 len 24576 08/30 Write block 351 len 8192 data: 13 12 09/30 Read block 479 len 40960 10/30 Write block 279 len 28672 data: 15 18 22 20 17 15 16 11/30 Write block 399 len 20480 data: 16 13 14 13 13 12/30 Write block 329 len 40960 data: 13 15 14 15 13 12 12 13 10 14 13/30 Read block 532 len 24576 14/30 Write block 220 len 20480 data: 9 9 8 10 10 15/30 Read block 430 len 20480 16/30 Flush 17/30 Write block 185 len 16384 data: 17 17 17 14 18/30 Flush 19/30 Write block 446 len 16384 data: 12 10 11 10 20/30 Write block 213 len 4096 data: 8 21/30 Read block 434 len 4096 22/30 Write block 477 len 4096 data: 12 23/30 Read block 322 len 28672 24/30 Flush 25/30 Write block 449 len 8192 data: 11 10 26/30 Write block 448 len 32768 data: 12 12 11 10 11 15 17 15 27/30 Write block 228 len 36864 data: 12 12 10 12 14 13 12 15 13 28/30 Read block 92 len 12288 29/30 Write block 417 len 32768 data: 11 13 10 12 13 11 12 15 30/30 Write block 374 len 8192 data: 13 14 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:04.243975671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1832} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 279 279 279 F F F 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 162 162 162 280 280 280 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 162 162 162 281 281 281 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 162 162 162 280 280 280 T T T 12 240-259 161 161 161 279 279 279 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 162 162 162 280 280 280 F F F 17 340-359 162 162 162 280 280 280 F F F 18 360-379 161 161 161 279 279 279 T T T 19 380-399 162 162 162 280 280 280 F F F 20 400-419 162 162 162 280 280 280 T T T 21 420-439 161 161 161 279 279 279 T T T 22 440-459 162 162 162 282 282 282 T T T 23 460-479 162 162 162 282 282 282 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 162, Max flush: 282 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 279 279 279 F F F 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 162 162 162 280 280 280 F F F 6 120-139 150 150 150 260 260 260 F F F 7 140-159 150 150 150 260 260 260 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 162 162 162 281 281 281 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 163 163 163 283 283 283 F F F 12 240-259 161 161 161 279 279 279 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 162 162 162 280 280 280 F F F 17 340-359 162 162 162 280 280 280 F F F 18 360-379 163 163 163 283 283 283 F F F 19 380-399 162 162 162 280 280 280 F F F 20 400-419 163 163 163 283 283 283 F F F 21 420-439 163 163 163 283 283 283 F F F 22 440-459 163 163 163 283 283 283 F F F 23 460-479 162 162 162 282 282 282 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 F F F 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 163, Max flush: 283 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.517450872Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.561152705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562033744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562080464Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562096101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562111397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} {"msg":"Crucible c4c56159-cb63-43fa-80ed-2801b562e66f has session id: ce311261-fdb1-46d8-acd5-594eb688df81","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562708818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Upstairs opts: Upstairs UUID: c4c56159-cb63-43fa-80ed-2801b562e66f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562772104Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Crucible stats registered with UUID: c4c56159-cb63-43fa-80ed-2801b562e66f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.562796819Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:08.562810286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563138365Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563164599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} {"msg":"c4c56159-cb63-43fa-80ed-2801b562e66f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563244292Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563262518Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563285303Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563301759Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563328774Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563354598Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563383971Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563411505Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.56343741Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563461835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"ds_connection connected from Ok(127.0.0.1:57010)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.56348499Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:08.563590947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:08.563631858Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:08.563720998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"ds_connection connected from Ok(127.0.0.1:32782)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.563740534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.56385029Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.564237437Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.564272459Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.564768372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.564880018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.565295077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:08.565462111Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.565352617Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"ds_connection connected from Ok(127.0.0.1:45342)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.565492687Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.566517575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567027934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[0]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 280, 260, 260, 280, 281, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.56760094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[0]R generation[0..12]: [161, 159, 158, 152, 154, 162, 150, 150, 162, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567625105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567640471Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[1]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 280, 260, 260, 280, 281, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567654208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[1]R generation[0..12]: [161, 159, 158, 152, 154, 162, 150, 150, 162, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567667695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567681172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[2]R flush_numbers[0..12]: [279, 275, 274, 265, 267, 280, 260, 260, 280, 281, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.56769452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[2]R generation[0..12]: [161, 159, 158, 152, 154, 162, 150, 150, 162, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567709366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567723174Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567736091Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567748758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567761095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567773592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"":"downstairs","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567791209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567812054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"0","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567825101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"1","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567837829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"client":"2","":"downstairs","session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567850646Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567862463Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"c4c56159-cb63-43fa-80ed-2801b562e66f is now active with session: ce311261-fdb1-46d8-acd5-594eb688df81","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.56787477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567886158Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840,"session_id":"ce311261-fdb1-46d8-acd5-594eb688df81"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:18.567898016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 20480 data: 10 12 13 11 14 02/30 Flush 03/30 Write block 511 len 4096 data: 11 04/30 Flush 05/30 Write block 421 len 4096 data: 14 06/30 Read block 402 len 28672 07/30 Write block 2 len 24576 data: 8 9 10 14 15 13 08/30 Read block 268 len 4096 09/30 Write block 326 len 4096 data: 9 10/30 Read block 103 len 28672 11/30 Write block 113 len 20480 data: 12 12 14 14 11 12/30 Read block 85 len 36864 13/30 Write block 128 len 20480 data: 9 8 7 6 7 14/30 Write block 186 len 36864 data: 18 18 15 15 18 16 21 15 12 15/30 Write block 130 len 40960 data: 8 7 8 9 12 13 11 9 8 11 16/30 Flush 17/30 Write block 242 len 12288 data: 21 16 14 18/30 Read block 576 len 28672 19/30 Read block 441 len 16384 20/30 Write block 145 len 24576 data: 11 14 13 9 12 12 21/30 Read block 477 len 4096 22/30 Write block 15 len 20480 data: 13 13 10 10 9 23/30 Read block 357 len 36864 24/30 Read block 147 len 36864 25/30 Write block 541 len 28672 data: 22 23 20 20 17 20 15 26/30 Write block 128 len 24576 data: 10 9 9 8 9 10 27/30 Read block 230 len 8192 28/30 Write block 1 len 28672 data: 7 9 10 11 15 16 14 29/30 Read block 160 len 28672 30/30 Write block 73 len 16384 data: 15 11 10 7 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:18.588139481Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1840} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 286 286 286 T T T 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 152 152 152 265 265 265 T T T 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 164 164 164 286 286 286 T T T 7 140-159 164 164 164 284 284 284 T T T 8 160-179 162 162 162 280 280 280 F F F 9 180-199 164 164 164 286 286 286 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 163 163 163 283 283 283 F F F 12 240-259 161 161 161 279 279 279 T T T 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 164 164 164 286 286 286 F F F 17 340-359 162 162 162 280 280 280 F F F 18 360-379 163 163 163 283 283 283 F F F 19 380-399 162 162 162 280 280 280 F F F 20 400-419 163 163 163 283 283 283 F F F 21 420-439 164 164 164 286 286 286 F F F 22 440-459 163 163 163 283 283 283 F F F 23 460-479 162 162 162 282 282 282 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 158 158 158 272 272 272 T T T 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 164, Max flush: 286 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 287 287 287 F F F 1 020-039 159 159 159 275 275 275 F F F 2 040-059 158 158 158 274 274 274 F F F 3 060-079 165 165 165 287 287 287 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 165 165 165 287 287 287 F F F 7 140-159 165 165 165 287 287 287 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 164 164 164 286 286 286 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 163 163 163 283 283 283 F F F 12 240-259 165 165 165 287 287 287 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 164 164 164 286 286 286 F F F 17 340-359 162 162 162 280 280 280 F F F 18 360-379 163 163 163 283 283 283 F F F 19 380-399 162 162 162 280 280 280 F F F 20 400-419 163 163 163 283 283 283 F F F 21 420-439 164 164 164 286 286 286 F F F 22 440-459 163 163 163 283 283 283 F F F 23 460-479 162 162 162 282 282 282 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 154 154 154 267 267 267 F F F 27 540-559 165 165 165 287 287 287 F F F 28 560-579 160 160 160 276 276 276 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 165, Max flush: 287 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.834371062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:22.876513703Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.87694439Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.907788298Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.908539366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.908579397Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.908593934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.908608591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} {"msg":"Crucible e0a33725-3a69-44d2-b03f-0bf13e364e54 has session id: 733f7b21-c1d7-4574-97df-4569f4126988","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.90925695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Upstairs opts: Upstairs UUID: e0a33725-3a69-44d2-b03f-0bf13e364e54, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.9093502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Crucible stats registered with UUID: e0a33725-3a69-44d2-b03f-0bf13e364e54","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909374045Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:22.909391731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909799333Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909823427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} {"msg":"e0a33725-3a69-44d2-b03f-0bf13e364e54 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909894922Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909911638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909933953Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.90994731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.909970965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.90999575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910021234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910047559Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910075403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910103877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"ds_connection connected from Ok(127.0.0.1:55917)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910140719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"ds_connection connected from Ok(127.0.0.1:62999)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910166693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"ds_connection connected from Ok(127.0.0.1:60983)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910214103Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.910984416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.911064928Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.91110191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.91165604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.911714768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.911839391Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.91225537Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912322686Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[0]R flush_numbers[0..12]: [287, 275, 274, 287, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912439411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[0]R generation[0..12]: [165, 159, 158, 165, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912458877Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912473103Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[1]R flush_numbers[0..12]: [287, 275, 274, 287, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.91248648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[1]R generation[0..12]: [165, 159, 158, 165, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912500157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912519133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[2]R flush_numbers[0..12]: [287, 275, 274, 287, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.91253357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[2]R generation[0..12]: [165, 159, 158, 165, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912547047Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912560624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912573172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912592297Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912605215Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912617962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"":"downstairs","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912630809Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912642477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"0","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912655324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"1","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912668101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"client":"2","":"downstairs","session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912680948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2025-01-23T17:45:22.912696535Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"e0a33725-3a69-44d2-b03f-0bf13e364e54 is now active with session: 733f7b21-c1d7-4574-97df-4569f4126988","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912709172Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912723359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848,"session_id":"733f7b21-c1d7-4574-97df-4569f4126988"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:22.912735446Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 12288 data: 16 14 15 02/30 Read block 164 len 24576 03/30 Write block 465 len 24576 data: 10 11 13 13 10 12 04/30 Read block 12 len 12288 05/30 Read block 273 len 4096 06/30 Write block 405 len 32768 data: 9 9 12 13 15 15 19 17 07/30 Write block 533 len 16384 data: 18 17 16 13 08/30 Flush 09/30 Write block 447 len 28672 data: 11 13 13 12 11 12 16 10/30 Write block 547 len 36864 data: 16 16 12 10 9 9 10 12 11 11/30 Read block 241 len 36864 12/30 Read block 45 len 20480 13/30 Read block 389 len 24576 14/30 Write block 557 len 20480 data: 11 11 11 10 9 15/30 Write block 269 len 12288 data: 17 17 16 16/30 Read block 369 len 32768 17/30 Flush 18/30 Write block 67 len 12288 data: 12 12 14 19/30 Write block 322 len 20480 data: 9 9 10 11 10 20/30 Write block 36 len 16384 data: 15 13 14 12 21/30 Write block 489 len 36864 data: 15 13 11 10 7 10 8 11 11 22/30 Flush 23/30 Write block 8 len 28672 data: 13 11 16 16 17 16 15 24/30 Read block 514 len 8192 25/30 Read block 332 len 28672 26/30 Write block 339 len 16384 data: 15 16 13 15 27/30 Write block 414 len 32768 data: 14 13 10 12 14 11 13 15 28/30 Read block 263 len 4096 29/30 Write block 46 len 24576 data: 12 11 12 11 12 11 30/30 Write block 371 len 28672 data: 16 16 14 14 15 12 9 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:22.931757576Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1848} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 287 287 287 T T T 1 020-039 166 166 166 290 290 290 F F F 2 040-059 158 158 158 274 274 274 T T T 3 060-079 166 166 166 290 290 290 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 165 165 165 287 287 287 F F F 7 140-159 165 165 165 287 287 287 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 164 164 164 286 286 286 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 163 163 163 283 283 283 F F F 12 240-259 165 165 165 287 287 287 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 166 166 166 290 290 290 T T T 17 340-359 162 162 162 280 280 280 T T T 18 360-379 163 163 163 283 283 283 T T T 19 380-399 162 162 162 280 280 280 F F F 20 400-419 166 166 166 288 288 288 T T T 21 420-439 166 166 166 288 288 288 T T T 22 440-459 166 166 166 289 289 289 F F F 23 460-479 166 166 166 288 288 288 F F F 24 480-499 166 166 166 290 290 290 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 166 166 166 288 288 288 F F F 27 540-559 166 166 166 289 289 289 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 166, Max flush: 290 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 166 166 166 290 290 290 F F F 2 040-059 167 167 167 291 291 291 F F F 3 060-079 166 166 166 290 290 290 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 165 165 165 287 287 287 F F F 7 140-159 165 165 165 287 287 287 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 164 164 164 286 286 286 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 163 163 163 283 283 283 F F F 12 240-259 165 165 165 287 287 287 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 167 167 167 291 291 291 F F F 17 340-359 167 167 167 291 291 291 F F F 18 360-379 167 167 167 291 291 291 F F F 19 380-399 162 162 162 280 280 280 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 167 167 167 291 291 291 F F F 22 440-459 166 166 166 289 289 289 F F F 23 460-479 166 166 166 288 288 288 F F F 24 480-499 166 166 166 290 290 290 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 166 166 166 288 288 288 F F F 27 540-559 166 166 166 289 289 289 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 167, Max flush: 291 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.211159483Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:27.253823351Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.25438299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.285465056Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286158146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286198737Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286213084Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286227881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} {"msg":"Crucible ca885138-b8f4-4b00-b900-e89631962504 has session id: d98ef1f5-ba09-44a7-9f56-df32ae6a30d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286826331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Upstairs opts: Upstairs UUID: ca885138-b8f4-4b00-b900-e89631962504, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286874291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Crucible stats registered with UUID: ca885138-b8f4-4b00-b900-e89631962504","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.286899695Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:27.286921241Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287304468Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287331562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} {"msg":"ca885138-b8f4-4b00-b900-e89631962504 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287395058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287412894Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287436789Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287460944Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287488988Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287520771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287583458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.28761504Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287643254Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287667599Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35852)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287694663Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:52019)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287720098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39924)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.287815547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.288410208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.288439692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.288461947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289119934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289263743Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289288228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.28969431Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289813144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[0]R flush_numbers[0..12]: [291, 290, 291, 290, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289954354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[0]R generation[0..12]: [167, 166, 167, 166, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.28997365Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.289988157Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[1]R flush_numbers[0..12]: [291, 290, 291, 290, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290002054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[1]R generation[0..12]: [167, 166, 167, 166, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290015691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290029317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[2]R flush_numbers[0..12]: [291, 290, 291, 290, 267, 286, 287, 287, 280, 286, 282, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290042735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[2]R generation[0..12]: [167, 166, 167, 166, 154, 164, 165, 165, 162, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290055942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290069469Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290082156Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290095213Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:45:27.290107871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290129266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"":"downstairs","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290142223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290153851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"0","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290166978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"1","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290179565Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"client":"2","":"downstairs","session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290192432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.29020427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"ca885138-b8f4-4b00-b900-e89631962504 is now active with session: d98ef1f5-ba09-44a7-9f56-df32ae6a30d6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290216807Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290228354Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856,"session_id":"d98ef1f5-ba09-44a7-9f56-df32ae6a30d6"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:27.290240222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 136 len 32768 data: 12 10 9 12 11 9 10 8 02/30 Write block 541 len 16384 data: 23 24 21 21 03/30 Read block 193 len 8192 04/30 Read block 476 len 36864 05/30 Read block 524 len 12288 06/30 Read block 454 len 12288 07/30 Write block 195 len 8192 data: 13 8 08/30 Read block 52 len 36864 09/30 Write block 41 len 32768 data: 13 10 8 10 12 13 12 13 10/30 Write block 136 len 12288 data: 13 11 10 11/30 Flush 12/30 Flush 13/30 Write block 59 len 32768 data: 9 12 16 16 12 11 10 11 14/30 Write block 382 len 16384 data: 11 11 12 10 15/30 Read block 231 len 36864 16/30 Write block 26 len 20480 data: 11 13 15 14 12 17/30 Write block 56 len 24576 data: 9 10 10 10 13 17 18/30 Write block 440 len 8192 data: 7 6 19/30 Read block 538 len 4096 20/30 Read block 159 len 28672 21/30 Read block 121 len 36864 22/30 Read block 473 len 20480 23/30 Write block 227 len 28672 data: 11 13 13 11 13 15 14 24/30 Flush 25/30 Write block 532 len 12288 data: 18 19 18 26/30 Read block 146 len 32768 27/30 Read block 558 len 4096 28/30 Write block 187 len 4096 data: 19 29/30 Read block 388 len 16384 30/30 Read block 191 len 24576 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:27.310004341Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1856} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 168 168 168 292 292 292 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 168 168 168 292 292 292 T T T 10 200-219 162 162 162 282 282 282 F F F 11 220-239 168 168 168 294 294 294 F F F 12 240-259 165 165 165 287 287 287 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 167 167 167 291 291 291 F F F 17 340-359 167 167 167 291 291 291 F F F 18 360-379 167 167 167 291 291 291 F F F 19 380-399 168 168 168 294 294 294 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 167 167 167 291 291 291 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 166 166 166 288 288 288 F F F 24 480-499 166 166 166 290 290 290 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 166 166 166 288 288 288 T T T 27 540-559 168 168 168 292 292 292 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 168, Max flush: 294 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 168 168 168 292 292 292 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 162 162 162 280 280 280 F F F 9 180-199 169 169 169 295 295 295 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 168 168 168 294 294 294 F F F 12 240-259 165 165 165 287 287 287 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 162 162 162 280 280 280 F F F 15 300-319 157 157 157 271 271 271 F F F 16 320-339 167 167 167 291 291 291 F F F 17 340-359 167 167 167 291 291 291 F F F 18 360-379 167 167 167 291 291 291 F F F 19 380-399 168 168 168 294 294 294 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 167 167 167 291 291 291 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 166 166 166 288 288 288 F F F 24 480-499 166 166 166 290 290 290 F F F 25 500-519 164 164 164 285 285 285 F F F 26 520-539 169 169 169 295 295 295 F F F 27 540-559 168 168 168 292 292 292 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 160 160 160 277 277 277 F F F Max gen: 169, Max flush: 295 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.57055001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:31.614043968Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.614615065Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.646633049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.647575865Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.647619095Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.647637911Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.647655997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} {"msg":"Crucible 015a22cf-db0f-44e6-b278-45536ca00ad3 has session id: aaaef94f-68bb-47ac-a362-1802ad079500","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648218345Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Upstairs opts: Upstairs UUID: 015a22cf-db0f-44e6-b278-45536ca00ad3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648264086Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Crucible stats registered with UUID: 015a22cf-db0f-44e6-b278-45536ca00ad3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648282871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:31.648303647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648636955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648666628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} {"msg":"015a22cf-db0f-44e6-b278-45536ca00ad3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648735493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.64875264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648777764Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648791711Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648805948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648818715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648832523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.64884544Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648859906Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648874513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"ds_connection connected from Ok(127.0.0.1:62927)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.648967243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"ds_connection connected from Ok(127.0.0.1:35209)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.649000956Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"ds_connection connected from Ok(127.0.0.1:43774)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.64902699Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.649621632Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.649785506Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.650034562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.650231149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.650316191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.650625904Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.65096517Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.650991525Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[0]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 280, 295, 282, 294]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.65119688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[0]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 162, 169, 162, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651225414Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651241411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[1]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 280, 295, 282, 294]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651255528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[1]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 162, 169, 162, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651269025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651282532Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[2]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 280, 295, 282, 294]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651300078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[2]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 162, 169, 162, 168]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651314075Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651328142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651347898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651360945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651374112Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651387049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"":"downstairs","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651400116Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651411534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"0","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651424591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"1","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651437438Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"client":"2","":"downstairs","session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651450545Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651462493Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"015a22cf-db0f-44e6-b278-45536ca00ad3 is now active with session: aaaef94f-68bb-47ac-a362-1802ad079500","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.65147491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:31.651486508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864,"session_id":"aaaef94f-68bb-47ac-a362-1802ad079500"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:45:31.651500785Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 550 len 4096 data: 11 02/30 Write block 566 len 28672 data: 15 14 12 14 11 10 10 03/30 Read block 532 len 8192 04/30 Flush 05/30 Write block 232 len 36864 data: 16 15 13 16 14 15 18 18 18 06/30 Write block 580 len 12288 data: 13 14 13 07/30 Read block 27 len 8192 08/30 Read block 408 len 24576 09/30 Write block 304 len 28672 data: 17 13 12 12 13 11 13 10/30 Read block 198 len 28672 11/30 Read block 260 len 12288 12/30 Read block 459 len 36864 13/30 Write block 358 len 4096 data: 10 14/30 Read block 535 len 4096 15/30 Write block 488 len 12288 data: 14 16 14 16/30 Read block 386 len 12288 17/30 Read block 293 len 36864 18/30 Write block 434 len 8192 data: 9 10 19/30 Write block 500 len 36864 data: 11 11 14 13 15 14 13 11 10 20/30 Flush 21/30 Write block 479 len 36864 data: 12 15 15 15 16 19 18 16 15 22/30 Write block 160 len 4096 data: 14 23/30 Write block 566 len 20480 data: 16 15 13 15 12 24/30 Read block 500 len 40960 25/30 Read block 256 len 4096 26/30 Read block 320 len 32768 27/30 Write block 395 len 20480 data: 15 14 13 16 17 28/30 Write block 292 len 24576 data: 14 13 16 16 17 13 29/30 Read block 144 len 40960 30/30 Read block 423 len 28672 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:31.671080833Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1864} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 168 168 168 292 292 292 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 162 162 162 280 280 280 T T T 9 180-199 169 169 169 295 295 295 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 170 170 170 297 297 297 F F F 12 240-259 170 170 170 297 297 297 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 162 162 162 280 280 280 T T T 15 300-319 170 170 170 297 297 297 F F F 16 320-339 167 167 167 291 291 291 F F F 17 340-359 170 170 170 297 297 297 F F F 18 360-379 167 167 167 291 291 291 F F F 19 380-399 168 168 168 294 294 294 T T T 20 400-419 167 167 167 291 291 291 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 166 166 166 288 288 288 T T T 24 480-499 170 170 170 297 297 297 T T T 25 500-519 170 170 170 297 297 297 F F F 26 520-539 169 169 169 295 295 295 F F F 27 540-559 170 170 170 296 296 296 F F F 28 560-579 170 170 170 296 296 296 T T T 29 580-599 170 170 170 297 297 297 F F F Max gen: 170, Max flush: 297 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 168 168 168 292 292 292 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 171 171 171 298 298 298 F F F 9 180-199 169 169 169 295 295 295 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 170 170 170 297 297 297 F F F 12 240-259 170 170 170 297 297 297 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 171 171 171 298 298 298 F F F 15 300-319 170 170 170 297 297 297 F F F 16 320-339 167 167 167 291 291 291 F F F 17 340-359 170 170 170 297 297 297 F F F 18 360-379 167 167 167 291 291 291 F F F 19 380-399 171 171 171 298 298 298 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 171 171 171 298 298 298 F F F 24 480-499 171 171 171 298 298 298 F F F 25 500-519 170 170 170 297 297 297 F F F 26 520-539 169 169 169 295 295 295 F F F 27 540-559 170 170 170 296 296 296 F F F 28 560-579 171 171 171 298 298 298 F F F 29 580-599 170 170 170 297 297 297 F F F Max gen: 171, Max flush: 298 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.951857071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.99839238Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.999486963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.9995447Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.999569835Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:35.999588881Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} {"msg":"Crucible 76ea99b0-9a7f-43f7-b4ca-fafc0e861b08 has session id: e9516ad1-487c-44bd-a506-1b20f2fef003","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000275792Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Upstairs opts: Upstairs UUID: 76ea99b0-9a7f-43f7-b4ca-fafc0e861b08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000344078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Crucible stats registered with UUID: 76ea99b0-9a7f-43f7-b4ca-fafc0e861b08","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000370412Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:36.000384309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000773464Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000798159Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} {"msg":"76ea99b0-9a7f-43f7-b4ca-fafc0e861b08 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000862635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000880791Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000904046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000918273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.00093229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000945467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000959184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000975801Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.000989738Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.001002905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"ds_connection connected from Ok(127.0.0.1:47940)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.001505046Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"ds_connection connected from Ok(127.0.0.1:48101)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.001544278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"ds_connection connected from Ok(127.0.0.1:64940)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.001569452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.002145407Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.002187318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.002220811Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.00277918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.002807664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.002894575Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003429999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003542834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[0]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 298, 295, 282, 297]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003618108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[0]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 171, 169, 162, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003640533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.00365594Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[1]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 298, 295, 282, 297]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003670017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[1]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 171, 169, 162, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003683514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003697181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[2]R flush_numbers[0..12]: [291, 294, 294, 294, 267, 286, 292, 292, 298, 295, 282, 297]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003710508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[2]R generation[0..12]: [167, 168, 168, 168, 154, 164, 168, 168, 171, 169, 162, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003723785Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003737322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.00374988Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003764936Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003798169Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003823834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"":"downstairs","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003843309Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003855797Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"0","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003873543Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"1","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003896688Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"client":"2","":"downstairs","session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003921952Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003943948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"76ea99b0-9a7f-43f7-b4ca-fafc0e861b08 is now active with session: e9516ad1-487c-44bd-a506-1b20f2fef003","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003957475Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.003973701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872,"session_id":"e9516ad1-487c-44bd-a506-1b20f2fef003"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:36.004012283Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 20480 data: 15 14 14 13 14 02/30 Write block 339 len 28672 data: 16 17 14 16 17 15 18 03/30 Read block 38 len 4096 04/30 Flush 05/30 Write block 591 len 32768 data: 8 9 9 7 7 5 5 4 06/30 Read block 19 len 8192 07/30 Read block 256 len 32768 08/30 Write block 85 len 20480 data: 22 19 18 17 17 09/30 Flush 10/30 Write block 463 len 24576 data: 9 10 11 12 14 14 11/30 Write block 228 len 40960 data: 14 14 12 14 17 16 14 17 15 16 12/30 Read block 171 len 40960 13/30 Read block 210 len 8192 14/30 Write block 41 len 40960 data: 14 11 9 11 13 14 13 14 12 13 15/30 Write block 271 len 8192 data: 17 19 16/30 Write block 125 len 12288 data: 16 9 9 17/30 Flush 18/30 Write block 476 len 28672 data: 13 13 12 13 16 16 16 19/30 Read block 63 len 36864 20/30 Read block 112 len 24576 21/30 Write block 251 len 20480 data: 12 10 11 9 10 22/30 Read block 90 len 28672 23/30 Flush 24/30 Write block 316 len 8192 data: 9 8 25/30 Read block 254 len 24576 26/30 Write block 501 len 8192 data: 12 15 27/30 Write block 378 len 20480 data: 6 8 11 10 12 28/30 Read block 370 len 32768 29/30 Read block 15 len 40960 30/30 Write block 423 len 20480 data: 13 16 15 16 18 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:36.024119077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1872} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 172 172 172 301 301 301 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 172 172 172 300 300 300 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 172 172 172 299 299 299 F F F 9 180-199 169 169 169 295 295 295 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 172 172 172 301 301 301 F F F 12 240-259 172 172 172 302 302 302 F F F 13 260-279 172 172 172 301 301 301 F F F 14 280-299 171 171 171 298 298 298 F F F 15 300-319 170 170 170 297 297 297 T T T 16 320-339 172 172 172 299 299 299 F F F 17 340-359 172 172 172 299 299 299 F F F 18 360-379 167 167 167 291 291 291 T T T 19 380-399 171 171 171 298 298 298 T T T 20 400-419 167 167 167 291 291 291 F F F 21 420-439 170 170 170 297 297 297 T T T 22 440-459 168 168 168 294 294 294 F F F 23 460-479 172 172 172 302 302 302 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 170 170 170 297 297 297 T T T 26 520-539 169 169 169 295 295 295 F F F 27 540-559 170 170 170 296 296 296 F F F 28 560-579 171 171 171 298 298 298 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 172, Max flush: 302 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 172 172 172 301 301 301 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 172 172 172 300 300 300 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 168 168 168 292 292 292 F F F 8 160-179 172 172 172 299 299 299 F F F 9 180-199 169 169 169 295 295 295 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 172 172 172 301 301 301 F F F 12 240-259 172 172 172 302 302 302 F F F 13 260-279 172 172 172 301 301 301 F F F 14 280-299 171 171 171 298 298 298 F F F 15 300-319 173 173 173 303 303 303 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 172 172 172 299 299 299 F F F 18 360-379 173 173 173 303 303 303 F F F 19 380-399 173 173 173 303 303 303 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 172 172 172 302 302 302 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 173 173 173 303 303 303 F F F 26 520-539 169 169 169 295 295 295 F F F 27 540-559 170 170 170 296 296 296 F F F 28 560-579 171 171 171 298 298 298 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 173, Max flush: 303 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.266411442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.310862683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.311640794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.311686554Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.311702981Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.311718208Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} {"msg":"Crucible 4bcac871-a03d-4cb5-9ab5-e6dc0924862f has session id: aae9f153-12aa-453e-b712-b2c8c5d5ea1f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.31230781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Upstairs opts: Upstairs UUID: 4bcac871-a03d-4cb5-9ab5-e6dc0924862f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.31235373Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Crucible stats registered with UUID: 4bcac871-a03d-4cb5-9ab5-e6dc0924862f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.312379644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:40.312404219Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.312887914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.312911739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} {"msg":"4bcac871-a03d-4cb5-9ab5-e6dc0924862f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.312972236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.3129993Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313029013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.31304453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313059767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313073044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313087101Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313099828Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313123233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313145848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54203)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313264602Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54656)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313312142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49143)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.313464149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314072627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314100871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314284781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314829783Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314868945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.314902168Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315438811Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315473834Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[0]R flush_numbers[0..12]: [291, 294, 301, 294, 300, 286, 301, 292, 299, 295, 282, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315542009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[0]R generation[0..12]: [167, 168, 172, 168, 172, 164, 172, 168, 172, 169, 162, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315569693Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.31558534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[1]R flush_numbers[0..12]: [291, 294, 301, 294, 300, 286, 301, 292, 299, 295, 282, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315599497Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[1]R generation[0..12]: [167, 168, 172, 168, 172, 164, 172, 168, 172, 169, 162, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315613094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315627171Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[2]R flush_numbers[0..12]: [291, 294, 301, 294, 300, 286, 301, 292, 299, 295, 282, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315640938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[2]R generation[0..12]: [167, 168, 172, 168, 172, 164, 172, 168, 172, 169, 162, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315654685Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315668322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:45:40.315684368Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315701085Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315713672Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315726179Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"":"downstairs","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315739426Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315751164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"0","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315764191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"1","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315777108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"client":"2","":"downstairs","session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315790035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315802203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"4bcac871-a03d-4cb5-9ab5-e6dc0924862f is now active with session: aae9f153-12aa-453e-b712-b2c8c5d5ea1f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.3158148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315826487Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880,"session_id":"aae9f153-12aa-453e-b712-b2c8c5d5ea1f"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:40.315838615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 12288 data: 16 12 14 02/30 Read block 398 len 40960 03/30 Flush 04/30 Write block 555 len 24576 data: 12 10 12 12 12 11 05/30 Read block 173 len 20480 06/30 Write block 289 len 16384 data: 14 11 15 15 07/30 Write block 248 len 36864 data: 10 10 12 13 11 12 10 11 9 08/30 Write block 464 len 24576 data: 11 12 13 15 15 11 09/30 Write block 78 len 16384 data: 9 9 12 12 10/30 Read block 142 len 8192 11/30 Write block 140 len 12288 data: 12 10 11 12/30 Write block 388 len 20480 data: 11 10 10 12 14 13/30 Read block 137 len 12288 14/30 Read block 239 len 12288 15/30 Flush 16/30 Write block 532 len 4096 data: 19 17/30 Read block 490 len 20480 18/30 Read block 379 len 28672 19/30 Read block 414 len 24576 20/30 Write block 533 len 4096 data: 20 21/30 Write block 265 len 4096 data: 16 22/30 Read block 206 len 16384 23/30 Write block 300 len 16384 data: 13 16 17 17 24/30 Write block 391 len 24576 data: 13 15 13 17 16 15 25/30 Read block 349 len 32768 26/30 Write block 381 len 28672 data: 11 13 12 13 11 10 12 27/30 Write block 547 len 24576 data: 17 17 13 12 10 10 28/30 Write block 157 len 16384 data: 16 16 17 16 29/30 Write block 343 len 4096 data: 18 30/30 Write block 53 len 20480 data: 11 9 9 10 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:40.333832707Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1880} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 172 172 172 301 301 301 T T T 3 060-079 174 174 174 305 305 305 F F F 4 080-099 174 174 174 305 305 305 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 174 174 174 305 305 305 T T T 8 160-179 172 172 172 299 299 299 T T T 9 180-199 169 169 169 295 295 295 F F F 10 200-219 174 174 174 304 304 304 F F F 11 220-239 172 172 172 301 301 301 F F F 12 240-259 174 174 174 305 305 305 F F F 13 260-279 172 172 172 301 301 301 T T T 14 280-299 174 174 174 305 305 305 F F F 15 300-319 173 173 173 303 303 303 T T T 16 320-339 172 172 172 299 299 299 F F F 17 340-359 172 172 172 299 299 299 T T T 18 360-379 173 173 173 303 303 303 F F F 19 380-399 174 174 174 305 305 305 T T T 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 174 174 174 305 305 305 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 173 173 173 303 303 303 F F F 26 520-539 169 169 169 295 295 295 T T T 27 540-559 174 174 174 305 305 305 T T T 28 560-579 174 174 174 305 305 305 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 174, Max flush: 305 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 174 174 174 305 305 305 F F F 4 080-099 174 174 174 305 305 305 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 175 175 175 306 306 306 F F F 8 160-179 175 175 175 306 306 306 F F F 9 180-199 169 169 169 295 295 295 F F F 10 200-219 174 174 174 304 304 304 F F F 11 220-239 172 172 172 301 301 301 F F F 12 240-259 174 174 174 305 305 305 F F F 13 260-279 175 175 175 306 306 306 F F F 14 280-299 174 174 174 305 305 305 F F F 15 300-319 175 175 175 306 306 306 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 175 175 175 306 306 306 F F F 18 360-379 173 173 173 303 303 303 F F F 19 380-399 175 175 175 306 306 306 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 174 174 174 305 305 305 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 173 173 173 303 303 303 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 175 175 175 306 306 306 F F F 28 560-579 174 174 174 305 305 305 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 175, Max flush: 306 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.598637386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:44.641160934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.64164162Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.672971253Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.673684659Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.673719781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.673735728Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.673753014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} {"msg":"Crucible 9656fdb4-4342-43c0-b067-43a316483dc0 has session id: b0c9e4ec-2d38-47bf-82fb-eb3691cb5151","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674353074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Upstairs opts: Upstairs UUID: 9656fdb4-4342-43c0-b067-43a316483dc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674408012Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Crucible stats registered with UUID: 9656fdb4-4342-43c0-b067-43a316483dc0","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674433027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:44.674449083Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674825701Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674855175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} {"msg":"9656fdb4-4342-43c0-b067-43a316483dc0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674884339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674897376Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674919911Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674933708Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674947905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674961242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.674984507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675009672Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675039265Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675066959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"ds_connection connected from Ok(127.0.0.1:48917)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675099982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"ds_connection connected from Ok(127.0.0.1:64987)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675124607Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"ds_connection connected from Ok(127.0.0.1:52820)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675258558Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.675944039Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.676023982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.676044677Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.676617673Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.676760193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.676798724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677177062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.67727769Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[0]R flush_numbers[0..12]: [291, 294, 306, 305, 305, 286, 301, 306, 306, 295, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677411671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[0]R generation[0..12]: [167, 168, 175, 174, 174, 164, 172, 175, 175, 169, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677433836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677448563Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[1]R flush_numbers[0..12]: [291, 294, 306, 305, 305, 286, 301, 306, 306, 295, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.67746209Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[1]R generation[0..12]: [167, 168, 175, 174, 174, 164, 172, 175, 175, 169, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677475987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677490154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[2]R flush_numbers[0..12]: [291, 294, 306, 305, 305, 286, 301, 306, 306, 295, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677504001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[2]R generation[0..12]: [167, 168, 175, 174, 174, 164, 172, 175, 175, 169, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677517698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677531195Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-01-23T17:45:44.677543873Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677563499Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677575696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677588113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"":"downstairs","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.67760093Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677612818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"0","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677625615Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"1","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677640082Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"client":"2","":"downstairs","session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677652919Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677664657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"9656fdb4-4342-43c0-b067-43a316483dc0 is now active with session: b0c9e4ec-2d38-47bf-82fb-eb3691cb5151","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677677064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:45:44.677688452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888,"session_id":"b0c9e4ec-2d38-47bf-82fb-eb3691cb5151"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:44.677703018Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 24576 data: 12 13 16 12 11 8 02/30 Read block 90 len 8192 03/30 Read block 284 len 20480 04/30 Read block 322 len 40960 05/30 Write block 201 len 16384 data: 10 15 14 14 06/30 Flush 07/30 Write block 296 len 36864 data: 18 14 13 10 14 17 18 18 18 08/30 Read block 407 len 36864 09/30 Read block 521 len 16384 10/30 Write block 223 len 28672 data: 11 11 9 12 12 15 15 11/30 Read block 374 len 16384 12/30 Write block 209 len 8192 data: 13 15 13/30 Read block 399 len 28672 14/30 Read block 22 len 8192 15/30 Read block 300 len 40960 16/30 Read block 501 len 16384 17/30 Write block 352 len 12288 data: 13 14 12 18/30 Read block 139 len 36864 19/30 Read block 18 len 40960 20/30 Read block 206 len 28672 21/30 Flush 22/30 Write block 288 len 4096 data: 15 23/30 Write block 559 len 36864 data: 13 12 10 10 8 11 14 17 16 24/30 Read block 506 len 36864 25/30 Write block 82 len 8192 data: 15 14 26/30 Write block 244 len 24576 data: 15 13 10 10 11 11 27/30 Read block 59 len 20480 28/30 Write block 183 len 28672 data: 17 17 18 19 20 16 16 29/30 Read block 165 len 24576 30/30 Read block 297 len 36864 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:44.699009973Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1888} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 174 174 174 305 305 305 T T T 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 175 175 175 306 306 306 F F F 8 160-179 175 175 175 306 306 306 F F F 9 180-199 169 169 169 295 295 295 T T T 10 200-219 176 176 176 308 308 308 F F F 11 220-239 176 176 176 308 308 308 F F F 12 240-259 174 174 174 305 305 305 T T T 13 260-279 175 175 175 306 306 306 F F F 14 280-299 176 176 176 308 308 308 T T T 15 300-319 176 176 176 308 308 308 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 176 176 176 308 308 308 F F F 18 360-379 173 173 173 303 303 303 F F F 19 380-399 175 175 175 306 306 306 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 174 174 174 305 305 305 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 173 173 173 303 303 303 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 175 175 175 306 306 306 T T T 28 560-579 174 174 174 305 305 305 T T T 29 580-599 172 172 172 300 300 300 F F F Max gen: 176, Max flush: 308 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 291 291 291 F F F 1 020-039 168 168 168 294 294 294 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 177 177 177 309 309 309 F F F 5 100-119 164 164 164 286 286 286 F F F 6 120-139 172 172 172 301 301 301 F F F 7 140-159 175 175 175 306 306 306 F F F 8 160-179 175 175 175 306 306 306 F F F 9 180-199 177 177 177 309 309 309 F F F 10 200-219 176 176 176 308 308 308 F F F 11 220-239 176 176 176 308 308 308 F F F 12 240-259 177 177 177 309 309 309 F F F 13 260-279 175 175 175 306 306 306 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 176 176 176 308 308 308 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 176 176 176 308 308 308 F F F 18 360-379 173 173 173 303 303 303 F F F 19 380-399 175 175 175 306 306 306 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 174 174 174 305 305 305 F F F 24 480-499 172 172 172 302 302 302 F F F 25 500-519 173 173 173 303 303 303 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 177 177 177 309 309 309 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 177, Max flush: 309 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:48.984902342Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:49.028981913Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.029506529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.061168071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.061972616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.062012278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.062027105Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.06205031Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} {"msg":"Crucible 024759dc-8ee5-4284-a7e0-78a430191aa9 has session id: 6b9dbcf8-21ea-4921-a010-06b77367615f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.062712646Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Upstairs opts: Upstairs UUID: 024759dc-8ee5-4284-a7e0-78a430191aa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.062747019Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Crucible stats registered with UUID: 024759dc-8ee5-4284-a7e0-78a430191aa9","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.062765845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:49.06278895Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063219656Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063244871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} {"msg":"024759dc-8ee5-4284-a7e0-78a430191aa9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063301899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063333982Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063359106Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063376263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063401997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063423692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063447357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063472352Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063500286Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.06352794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34050)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063556304Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38377)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063585257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48934)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.063771987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.064322008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.064362319Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.064446421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.064856812Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.064972527Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065265893Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065411891Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065496233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[0]R flush_numbers[0..12]: [291, 294, 306, 307, 309, 286, 301, 306, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065797658Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[0]R generation[0..12]: [167, 168, 175, 176, 177, 164, 172, 175, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065821243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065836479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[1]R flush_numbers[0..12]: [291, 294, 306, 307, 309, 286, 301, 306, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065850596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[1]R generation[0..12]: [167, 168, 175, 176, 177, 164, 172, 175, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065864453Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.06587861Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[2]R flush_numbers[0..12]: [291, 294, 306, 307, 309, 286, 301, 306, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065892458Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[2]R generation[0..12]: [167, 168, 175, 176, 177, 164, 172, 175, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065906064Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065922961Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:45:49.065935938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065951934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065964442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065976739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"":"downstairs","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.065989676Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066001494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"0","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066014691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"1","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066027718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"client":"2","":"downstairs","session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066040965Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066053113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"024759dc-8ee5-4284-a7e0-78a430191aa9 is now active with session: 6b9dbcf8-21ea-4921-a010-06b77367615f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.06606551Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066076987Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896,"session_id":"6b9dbcf8-21ea-4921-a010-06b77367615f"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:49.066088775Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 87 len 16384 data: 19 18 18 13 02/30 Read block 288 len 16384 03/30 Write block 126 len 16384 data: 10 10 11 10 04/30 Write block 384 len 12288 data: 14 12 11 05/30 Write block 483 len 40960 data: 17 20 19 17 16 15 17 15 12 11 06/30 Write block 118 len 20480 data: 14 17 17 17 17 07/30 Read block 223 len 16384 08/30 Write block 303 len 24576 data: 19 19 14 13 13 14 09/30 Write block 382 len 24576 data: 14 13 15 13 12 13 10/30 Write block 129 len 36864 data: 11 10 9 10 11 13 14 14 12 11/30 Read block 333 len 24576 12/30 Write block 259 len 40960 data: 12 13 11 12 12 14 17 18 19 20 13/30 Write block 144 len 28672 data: 9 12 15 14 10 13 13 14/30 Write block 380 len 24576 data: 12 12 15 14 16 14 15/30 Read block 414 len 36864 16/30 Flush 17/30 Write block 393 len 4096 data: 14 18/30 Write block 6 len 28672 data: 17 15 14 12 17 17 18 19/30 Read block 248 len 4096 20/30 Read block 6 len 8192 21/30 Flush 22/30 Write block 28 len 16384 data: 16 15 13 12 23/30 Read block 420 len 4096 24/30 Write block 25 len 4096 data: 11 25/30 Write block 461 len 32768 data: 11 14 10 12 13 14 16 16 26/30 Write block 504 len 40960 data: 16 15 14 12 11 11 12 12 9 9 27/30 Write block 265 len 8192 data: 18 19 28/30 Read block 520 len 40960 29/30 Write block 354 len 36864 data: 13 11 11 10 11 10 9 9 7 30/30 Read block 46 len 16384 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 7 9 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:49.085711324Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1896} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 311 311 311 F F F 1 020-039 168 168 168 294 294 294 T T T 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 178 178 178 310 310 310 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 175 175 175 306 306 306 F F F 9 180-199 177 177 177 309 309 309 F F F 10 200-219 176 176 176 308 308 308 F F F 11 220-239 176 176 176 308 308 308 F F F 12 240-259 178 178 178 310 310 310 F F F 13 260-279 178 178 178 310 310 310 T T T 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 176 176 176 308 308 308 T T T 18 360-379 173 173 173 303 303 303 T T T 19 380-399 178 178 178 311 311 311 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 174 174 174 305 305 305 T T T 24 480-499 178 178 178 310 310 310 F F F 25 500-519 173 173 173 303 303 303 T T T 26 520-539 175 175 175 306 306 306 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 177 177 177 309 309 309 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 178, Max flush: 311 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 311 311 311 F F F 1 020-039 179 179 179 312 312 312 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 178 178 178 310 310 310 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 175 175 175 306 306 306 F F F 9 180-199 177 177 177 309 309 309 F F F 10 200-219 176 176 176 308 308 308 F F F 11 220-239 176 176 176 308 308 308 F F F 12 240-259 178 178 178 310 310 310 F F F 13 260-279 179 179 179 312 312 312 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 179 179 179 312 312 312 F F F 19 380-399 178 178 178 311 311 311 F F F 20 400-419 167 167 167 291 291 291 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 179 179 179 312 312 312 F F F 24 480-499 178 178 178 310 310 310 F F F 25 500-519 179 179 179 312 312 312 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 177 177 177 309 309 309 F F F 29 580-599 172 172 172 300 300 300 F F F Max gen: 179, Max flush: 312 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.367446584Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:53.411924789Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.412587186Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.444588044Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.445460715Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.445505495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.445523181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.445547016Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} {"msg":"Crucible 634b0a88-2752-40a8-9bf8-ca86f171925f has session id: 12e46195-7c40-4e8e-8b5b-2c271565a3a6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446139868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Upstairs opts: Upstairs UUID: 634b0a88-2752-40a8-9bf8-ca86f171925f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446178009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Crucible stats registered with UUID: 634b0a88-2752-40a8-9bf8-ca86f171925f","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446194556Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:53.446208193Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446639319Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446663634Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} {"msg":"634b0a88-2752-40a8-9bf8-ca86f171925f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446732679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446750585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.44677322Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446787257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446802074Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446826299Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446854973Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446877228Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446904392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446935705Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59110)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.446964399Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54843)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.447178183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57009)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.447229032Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.447753628Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.447787631Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.44806534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.448541257Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.448570731Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.448586627Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449191256Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449217781Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[0]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449258542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[0]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449279057Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449293524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[1]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449307091Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[1]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449320419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449333885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[2]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 306, 309, 308, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449347153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[2]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 175, 177, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.44936058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449374167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449386844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449400001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449412768Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg"Wait for a query_work_queue command to finish before sending IO :"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449436853Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"":"downstairs","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449449571Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449461038Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"0","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449475655Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"1","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449488752Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"client":"2","":"downstairs","session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449501949Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449513837Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"634b0a88-2752-40a8-9bf8-ca86f171925f is now active with session: 12e46195-7c40-4e8e-8b5b-2c271565a3a6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449526114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449537671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904,"session_id":"12e46195-7c40-4e8e-8b5b-2c271565a3a6"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:53.449549549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 268 len 32768 data: 21 18 18 18 20 22 19 15 02/30 Read block 375 len 24576 03/30 Write block 375 len 8192 data: 16 13 04/30 Write block 506 len 24576 data: 15 13 12 12 13 13 05/30 Flush 06/30 Write block 554 len 12288 data: 13 13 11 07/30 Flush 08/30 Write block 403 len 32768 data: 14 12 10 10 13 14 16 16 09/30 Write block 166 len 24576 data: 11 10 11 13 14 15 10/30 Read block 273 len 20480 11/30 Read block 31 len 40960 12/30 Read block 24 len 20480 13/30 Read block 376 len 24576 14/30 Write block 206 len 28672 data: 15 16 17 14 16 15 13 15/30 Read block 122 len 36864 16/30 Read block 490 len 12288 17/30 Write block 180 len 28672 data: 14 13 16 18 18 19 20 18/30 Read block 521 len 32768 19/30 Read block 128 len 12288 20/30 Write block 208 len 36864 data: 18 15 17 16 14 9 8 8 10 21/30 Read block 503 len 12288 22/30 Read block 140 len 4096 23/30 Write block 207 len 40960 data: 17 19 16 18 17 15 10 9 9 11 24/30 Read block 208 len 8192 25/30 Write block 553 len 40960 data: 11 14 14 12 13 13 14 13 11 11 26/30 Read block 212 len 40960 27/30 Read block 93 len 8192 28/30 Read block 500 len 40960 29/30 Write block 592 len 4096 data: 10 30/30 Write block 482 len 32768 data: 17 18 21 20 18 17 16 18 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 8 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:53.472073459Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1904} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 311 311 311 F F F 1 020-039 179 179 179 312 312 312 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 178 178 178 310 310 310 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 175 175 175 306 306 306 T T T 9 180-199 177 177 177 309 309 309 T T T 10 200-219 176 176 176 308 308 308 T T T 11 220-239 176 176 176 308 308 308 F F F 12 240-259 178 178 178 310 310 310 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 180 180 180 313 313 313 F F F 19 380-399 178 178 178 311 311 311 F F F 20 400-419 167 167 167 291 291 291 T T T 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 179 179 179 312 312 312 F F F 24 480-499 178 178 178 310 310 310 T T T 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 180 180 180 314 314 314 T T T 28 560-579 177 177 177 309 309 309 T T T 29 580-599 172 172 172 300 300 300 T T T Max gen: 180, Max flush: 314 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 311 311 311 F F F 1 020-039 179 179 179 312 312 312 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 176 176 176 307 307 307 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 178 178 178 310 310 310 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 181 181 181 315 315 315 F F F 10 200-219 181 181 181 315 315 315 F F F 11 220-239 176 176 176 308 308 308 F F F 12 240-259 178 178 178 310 310 310 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 180 180 180 313 313 313 F F F 19 380-399 178 178 178 311 311 311 F F F 20 400-419 181 181 181 315 315 315 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 179 179 179 312 312 312 F F F 24 480-499 181 181 181 315 315 315 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 181 181 181 315 315 315 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 181, Max flush: 315 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.769790947Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.813819729Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.814789479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.814837039Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.814853625Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.814868532Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} {"msg":"Crucible edecbae5-78be-4a53-8b3f-f5845f0c1158 has session id: eab95173-e30e-4434-a06c-89019a87de7e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.815551014Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Upstairs opts: Upstairs UUID: edecbae5-78be-4a53-8b3f-f5845f0c1158, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.815600523Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Crucible stats registered with UUID: edecbae5-78be-4a53-8b3f-f5845f0c1158","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.815625418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:57.815647473Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816063023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816088217Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} {"msg":"edecbae5-78be-4a53-8b3f-f5845f0c1158 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.81616962Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816203383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816243194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816273388Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816304261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816334364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.81635645Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816370427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816384683Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.81639899Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45922)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816423185Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56380)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816452009Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38200)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.816480113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.817131442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.817163735Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.817755036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.817891807Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.817916261Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818367054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818504794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818588136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[0]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 315, 315, 315, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818855198Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[0]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 181, 181, 181, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818874804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818889491Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[1]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 315, 315, 315, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818903348Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[1]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 181, 181, 181, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818916765Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818930642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[2]R flush_numbers[0..12]: [311, 312, 306, 307, 310, 310, 310, 310, 315, 315, 315, 308]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818943979Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[2]R generation[0..12]: [178, 179, 175, 176, 178, 178, 178, 178, 181, 181, 181, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818961485Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818975442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.818988139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819006845Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819019542Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.81903199Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"":"downstairs","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819046147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819057844Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"0","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819070651Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"1","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819083479Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"client":"2","":"downstairs","session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819096166Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819107763Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"edecbae5-78be-4a53-8b3f-f5845f0c1158 is now active with session: eab95173-e30e-4434-a06c-89019a87de7e","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819120291Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819132128Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912,"session_id":"eab95173-e30e-4434-a06c-89019a87de7e"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:45:57.819144635Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 36864 data: 15 18 16 17 19 19 19 18 22 02/30 Read block 399 len 4096 03/30 Flush 04/30 Write block 452 len 8192 data: 13 17 05/30 Read block 505 len 32768 06/30 Read block 512 len 28672 07/30 Read block 527 len 28672 08/30 Flush 09/30 Write block 63 len 12288 data: 13 12 11 10/30 Read block 10 len 12288 11/30 Write block 565 len 12288 data: 15 18 17 12/30 Write block 210 len 24576 data: 19 18 16 11 10 10 13/30 Read block 305 len 16384 14/30 Flush 15/30 Write block 117 len 16384 data: 12 15 18 18 16/30 Read block 373 len 24576 17/30 Flush 18/30 Write block 370 len 40960 data: 15 17 17 15 15 17 14 10 7 9 19/30 Read block 270 len 36864 20/30 Write block 488 len 4096 data: 17 21/30 Write block 541 len 12288 data: 24 25 22 22/30 Read block 387 len 32768 23/30 Write block 29 len 4096 data: 16 24/30 Read block 81 len 16384 25/30 Read block 544 len 20480 26/30 Flush 27/30 Write block 412 len 28672 data: 18 12 15 14 11 13 15 28/30 Read block 155 len 16384 29/30 Write block 237 len 28672 data: 18 20 20 20 19 23 17 30/30 Write block 8 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:45:57.838981118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1912} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 311 311 311 T T T 1 020-039 182 182 182 320 320 320 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 182 182 182 318 318 318 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 182 182 182 319 319 319 F F F 6 120-139 182 182 182 319 319 319 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 181 181 181 315 315 315 F F F 10 200-219 182 182 182 318 318 318 F F F 11 220-239 182 182 182 316 316 316 T T T 12 240-259 182 182 182 316 316 316 T T T 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 182 182 182 320 320 320 F F F 19 380-399 178 178 178 311 311 311 F F F 20 400-419 181 181 181 315 315 315 T T T 21 420-439 173 173 173 303 303 303 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 179 179 179 312 312 312 F F F 24 480-499 182 182 182 320 320 320 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 182 182 182 320 320 320 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 182, Max flush: 320 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 321 321 321 F F F 1 020-039 182 182 182 320 320 320 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 182 182 182 318 318 318 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 182 182 182 319 319 319 F F F 6 120-139 182 182 182 319 319 319 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 181 181 181 315 315 315 F F F 10 200-219 182 182 182 318 318 318 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 183 183 183 321 321 321 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 178 178 178 310 310 310 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 182 182 182 320 320 320 F F F 19 380-399 178 178 178 311 311 311 F F F 20 400-419 183 183 183 321 321 321 F F F 21 420-439 173 173 173 303 303 303 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 179 179 179 312 312 312 F F F 24 480-499 182 182 182 320 320 320 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 182 182 182 320 320 320 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 183, Max flush: 321 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.078926501Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.122622855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.123352307Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.123396807Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.123411794Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.123426851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} {"msg":"Crucible c241939f-33bc-4cf2-be37-83c4d98799c5 has session id: 0b3b9d24-08e4-44fa-8fef-9a94ac58d007","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124069942Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Upstairs opts: Upstairs UUID: c241939f-33bc-4cf2-be37-83c4d98799c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124131619Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Crucible stats registered with UUID: c241939f-33bc-4cf2-be37-83c4d98799c5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124161332Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:02.124187236Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124646077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124673071Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} {"msg":"c241939f-33bc-4cf2-be37-83c4d98799c5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124736467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124754153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124776679Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124790996Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124805742Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124819339Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124833476Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124846793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124860421Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.124873948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"ds_connection connected from Ok(127.0.0.1:57318)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.125043001Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"ds_connection connected from Ok(127.0.0.1:61383)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.125099909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:02.125195678Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:02.125215893Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:02.12523204Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.125310113Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.12578403Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.125809985Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.126530419Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.126558773Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.12717235Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:02.127258851Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.126598918Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"ds_connection connected from Ok(127.0.0.1:51321)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.126685279Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.127775153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.128385181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[0]R flush_numbers[0..12]: [321, 320, 306, 318, 310, 319, 319, 310, 315, 315, 318, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.128953398Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[0]R generation[0..12]: [183, 182, 175, 182, 178, 182, 182, 178, 181, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.128980772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.128996638Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[1]R flush_numbers[0..12]: [321, 320, 306, 318, 310, 319, 319, 310, 315, 315, 318, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129011305Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[1]R generation[0..12]: [183, 182, 175, 182, 178, 182, 182, 178, 181, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129026222Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129040059Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[2]R flush_numbers[0..12]: [321, 320, 306, 318, 310, 319, 319, 310, 315, 315, 318, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129053536Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[2]R generation[0..12]: [183, 182, 175, 182, 178, 182, 182, 178, 181, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129067153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.12908121Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129095187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129108384Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.12912664Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129140037Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"":"downstairs","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129153264Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129165232Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"0","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129178359Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"client":"1","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129191196Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":Wait for a query_work_queue command to finish before sending IO 1920,"client":"2","":"downstairs","session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129215131Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129227088Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"c241939f-33bc-4cf2-be37-83c4d98799c5 is now active with session: 0b3b9d24-08e4-44fa-8fef-9a94ac58d007","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129239626Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129251133Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920,"session_id":"0b3b9d24-08e4-44fa-8fef-9a94ac58d007"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:12.129262941Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 32768 data: 11 8 10 13 13 16 15 17 02/30 Write block 408 len 20480 data: 15 17 17 20 19 03/30 Read block 559 len 12288 04/30 Read block 515 len 36864 05/30 Read block 276 len 32768 06/30 Read block 23 len 40960 07/30 Read block 457 len 16384 08/30 Write block 303 len 32768 data: 20 20 15 14 14 15 12 14 09/30 Read block 290 len 20480 10/30 Read block 15 len 12288 11/30 Write block 204 len 12288 data: 15 13 16 12/30 Flush 13/30 Write block 424 len 28672 data: 17 16 17 19 17 15 16 14/30 Write block 18 len 32768 data: 11 10 11 11 11 12 12 12 15/30 Read block 229 len 16384 16/30 Flush 17/30 Write block 120 len 36864 data: 19 18 18 16 16 17 11 11 12 18/30 Write block 540 len 8192 data: 20 25 19/30 Write block 542 len 32768 data: 26 23 22 18 21 18 18 14 20/30 Write block 563 len 20480 data: 9 12 16 19 18 21/30 Read block 334 len 8192 22/30 Read block 356 len 8192 23/30 Read block 309 len 16384 24/30 Write block 244 len 20480 data: 16 14 11 11 12 25/30 Flush 26/30 Write block 466 len 12288 data: 15 17 17 27/30 Write block 80 len 12288 data: 13 13 16 28/30 Write block 76 len 40960 data: 9 8 10 10 14 14 17 15 15 23 29/30 Read block 524 len 32768 30/30 Read block 296 len 24576 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:12.151572778Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1920} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 182 182 182 318 318 318 T T T 4 080-099 178 178 178 310 310 310 T T T 5 100-119 182 182 182 319 319 319 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 181 181 181 315 315 315 F F F 10 200-219 184 184 184 322 322 322 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 184 184 184 324 324 324 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 184 184 184 322 322 322 F F F 19 380-399 184 184 184 322 322 322 F F F 20 400-419 184 184 184 322 322 322 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 179 179 179 312 312 312 T T T 24 480-499 182 182 182 320 320 320 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 184, Max flush: 324 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 185 185 185 325 325 325 F F F 4 080-099 185 185 185 325 325 325 F F F 5 100-119 182 182 182 319 319 319 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 181 181 181 315 315 315 F F F 10 200-219 184 184 184 322 322 322 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 184 184 184 324 324 324 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 172 172 172 299 299 299 F F F 17 340-359 179 179 179 312 312 312 F F F 18 360-379 184 184 184 322 322 322 F F F 19 380-399 184 184 184 322 322 322 F F F 20 400-419 184 184 184 322 322 322 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 185 185 185 325 325 325 F F F 24 480-499 182 182 182 320 320 320 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 185, Max flush: 325 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.390620785Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.435944357Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.436868077Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.436915886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.436931063Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.43694613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} {"msg":"Crucible ab46819c-5ab5-4be5-9237-eebe3c3c7408 has session id: 8fd01dca-3540-4625-8b58-101836b6e7e6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.437645838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Upstairs opts: Upstairs UUID: ab46819c-5ab5-4be5-9237-eebe3c3c7408, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.437696137Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Crucible stats registered with UUID: ab46819c-5ab5-4be5-9237-eebe3c3c7408","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.437721562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:16.437736549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438243129Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438273302Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} {"msg":"ab46819c-5ab5-4be5-9237-eebe3c3c7408 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.43833043Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438357004Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438380739Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438395726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438409733Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438431938Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438459442Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438483507Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438508061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438535395Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62140)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438620317Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54659)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438652Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40884)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.438849807Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.439481011Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.439525181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.440341934Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.440374347Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.440771531Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441002441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441035744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.4413321Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 319, 324, 310, 315, 315, 322, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441854596Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[0]R generation[0..12]: [184, 184, 175, 185, 185, 182, 184, 178, 181, 181, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441876572Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441891808Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 319, 324, 310, 315, 315, 322, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441905616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[1]R generation[0..12]: [184, 184, 175, 185, 185, 182, 184, 178, 181, 181, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441919073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.44193252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 319, 324, 310, 315, 315, 322, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441947356Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[2]R generation[0..12]: [184, 184, 175, 185, 185, 182, 184, 178, 181, 181, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441966732Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441981189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.441993766Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442012372Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442025239Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442037907Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"":"downstairs","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442050914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442062382Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"0","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442075269Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"1","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442087896Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"client":"2","":"downstairs","session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442101053Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.44211306Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"ab46819c-5ab5-4be5-9237-eebe3c3c7408 is now active with session: 8fd01dca-3540-4625-8b58-101836b6e7e6","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442126328Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442137885Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928,"session_id":"8fd01dca-3540-4625-8b58-101836b6e7e6"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:16.442149923Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 40960 data: 13 11 12 9 10 13 16 15 11 14 02/30 Write block 105 len 28672 data: 12 13 12 12 11 13 15 03/30 Read block 493 len 16384 04/30 Read block 135 len 36864 05/30 Read block 396 len 40960 06/30 Write block 146 len 20480 data: 17 16 12 15 14 07/30 Flush 08/30 Write block 372 len 28672 data: 18 16 16 18 15 12 9 09/30 Read block 397 len 4096 10/30 Write block 478 len 20480 data: 13 14 17 17 18 11/30 Read block 588 len 36864 12/30 Flush 13/30 Write block 183 len 28672 data: 19 19 20 21 21 17 17 14/30 Write block 338 len 32768 data: 15 17 18 15 17 19 16 19 15/30 Write block 356 len 32768 data: 12 11 12 11 10 10 8 8 16/30 Read block 474 len 36864 17/30 Read block 566 len 32768 18/30 Flush 19/30 Write block 363 len 16384 data: 9 9 9 9 20/30 Read block 101 len 32768 21/30 Read block 452 len 20480 22/30 Read block 378 len 32768 23/30 Write block 144 len 24576 data: 11 14 18 17 13 16 24/30 Read block 35 len 24576 25/30 Write block 113 len 28672 data: 13 13 15 15 13 16 19 26/30 Read block 555 len 20480 27/30 Read block 313 len 16384 28/30 Write block 205 len 4096 data: 14 29/30 Read block 365 len 24576 30/30 Write block 404 len 28672 data: 13 11 11 14 16 18 18 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:16.464391245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1928} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 185 185 185 325 325 325 F F F 4 080-099 185 185 185 325 325 325 F F F 5 100-119 186 186 186 326 326 326 T T T 6 120-139 184 184 184 324 324 324 F F F 7 140-159 186 186 186 326 326 326 T T T 8 160-179 181 181 181 315 315 315 F F F 9 180-199 186 186 186 328 328 328 F F F 10 200-219 184 184 184 322 322 322 T T T 11 220-239 183 183 183 321 321 321 F F F 12 240-259 184 184 184 324 324 324 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 186 186 186 328 328 328 F F F 17 340-359 186 186 186 328 328 328 F F F 18 360-379 186 186 186 328 328 328 T T T 19 380-399 184 184 184 322 322 322 F F F 20 400-419 184 184 184 322 322 322 T T T 21 420-439 184 184 184 323 323 323 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 186 186 186 327 327 327 F F F 24 480-499 186 186 186 327 327 327 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 186, Max flush: 328 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 185 185 185 325 325 325 F F F 4 080-099 185 185 185 325 325 325 F F F 5 100-119 187 187 187 329 329 329 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 187 187 187 329 329 329 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 186 186 186 328 328 328 F F F 10 200-219 187 187 187 329 329 329 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 184 184 184 324 324 324 F F F 13 260-279 180 180 180 313 313 313 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 186 186 186 328 328 328 F F F 17 340-359 186 186 186 328 328 328 F F F 18 360-379 187 187 187 329 329 329 F F F 19 380-399 184 184 184 322 322 322 F F F 20 400-419 187 187 187 329 329 329 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 182 182 182 317 317 317 F F F 23 460-479 186 186 186 327 327 327 F F F 24 480-499 186 186 186 327 327 327 F F F 25 500-519 180 180 180 313 313 313 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 187, Max flush: 329 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.710286758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.754970898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.755727035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.755778533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.755796449Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.755818955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} {"msg":"Crucible 4b556f16-a8d2-44dd-85cd-aac4bfaa1bd7 has session id: 7229fd65-b498-4025-b659-5d28a7287a74","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756357098Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Upstairs opts: Upstairs UUID: 4b556f16-a8d2-44dd-85cd-aac4bfaa1bd7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756406637Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Crucible stats registered with UUID: 4b556f16-a8d2-44dd-85cd-aac4bfaa1bd7","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756431462Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:20.756452547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756842293Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756877955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} {"msg":"4b556f16-a8d2-44dd-85cd-aac4bfaa1bd7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756964736Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.756997829Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.75704017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757070364Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757099117Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757123612Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757151516Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.75717657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757207514Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757432115Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"ds_connection connected from Ok(127.0.0.1:43739)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757465448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"ds_connection connected from Ok(127.0.0.1:58770)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757493482Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"ds_connection connected from Ok(127.0.0.1:39623)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.757517347Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.75805547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.758082964Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.758097591Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.758685814Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.758715187Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.758776534Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759314787Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759421144Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 329, 324, 329, 315, 328, 329, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759491909Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[0]R generation[0..12]: [184, 184, 175, 185, 185, 187, 184, 187, 181, 186, 187, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759514194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759529061Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 329, 324, 329, 315, 328, 329, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759542668Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[1]R generation[0..12]: [184, 184, 175, 185, 185, 187, 184, 187, 181, 186, 187, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759556385Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759570122Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 306, 325, 325, 329, 324, 329, 315, 328, 329, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759583629Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[2]R generation[0..12]: [184, 184, 175, 185, 185, 187, 184, 187, 181, 186, 187, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759596756Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759610273Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759623051Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759635648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759648175Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759660622Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759681868Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759693665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"0","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759708552Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"1","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759721859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"client":"2","":"downstairs","session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759734726Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759746624Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"4b556f16-a8d2-44dd-85cd-aac4bfaa1bd7 is now active with session: 7229fd65-b498-4025-b659-5d28a7287a74","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759759171Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759770948Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936,"session_id":"7229fd65-b498-4025-b659-5d28a7287a74"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:20.759782886Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 330 len 4096 data: 16 02/30 Write block 117 len 40960 data: 14 17 20 20 19 19 17 17 18 12 03/30 Read block 520 len 16384 04/30 Read block 527 len 16384 05/30 Write block 412 len 32768 data: 20 13 16 15 12 14 16 12 06/30 Write block 143 len 8192 data: 10 12 07/30 Flush 08/30 Write block 274 len 24576 data: 20 16 13 14 16 16 09/30 Read block 51 len 32768 10/30 Read block 593 len 16384 11/30 Read block 491 len 36864 12/30 Read block 588 len 8192 13/30 Write block 446 len 16384 data: 13 12 14 14 14/30 Read block 516 len 8192 15/30 Read block 556 len 16384 16/30 Write block 68 len 20480 data: 13 15 12 13 14 17/30 Write block 186 len 24576 data: 22 22 18 18 19 17 18/30 Write block 183 len 40960 data: 20 20 21 23 23 19 19 20 18 22 19/30 Read block 326 len 24576 20/30 Read block 223 len 16384 21/30 Flush 22/30 Write block 391 len 16384 data: 14 16 15 18 23/30 Flush 24/30 Write block 215 len 20480 data: 11 12 10 10 12 25/30 Write block 499 len 20480 data: 10 12 13 16 14 26/30 Write block 290 len 40960 data: 12 16 16 14 17 17 19 15 14 11 27/30 Read block 578 len 24576 28/30 Write block 252 len 24576 data: 12 13 11 12 10 11 29/30 Write block 80 len 8192 data: 15 15 30/30 Read block 246 len 36864 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:20.779354266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1936} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 185 185 185 325 325 325 T T T 5 100-119 188 188 188 330 330 330 F F F 6 120-139 188 188 188 330 330 330 F F F 7 140-159 188 188 188 330 330 330 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 187 187 187 329 329 329 T T T 11 220-239 183 183 183 321 321 321 F F F 12 240-259 184 184 184 324 324 324 T T T 13 260-279 188 188 188 331 331 331 F F F 14 280-299 177 177 177 309 309 309 T T T 15 300-319 184 184 184 322 322 322 F F F 16 320-339 188 188 188 330 330 330 F F F 17 340-359 186 186 186 328 328 328 F F F 18 360-379 187 187 187 329 329 329 F F F 19 380-399 188 188 188 332 332 332 F F F 20 400-419 188 188 188 330 330 330 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 186 186 186 327 327 327 F F F 24 480-499 186 186 186 327 327 327 T T T 25 500-519 180 180 180 313 313 313 T T T 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 188, Max flush: 332 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 189 189 189 333 333 333 F F F 5 100-119 188 188 188 330 330 330 F F F 6 120-139 188 188 188 330 330 330 F F F 7 140-159 188 188 188 330 330 330 F F F 8 160-179 181 181 181 315 315 315 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 189 189 189 333 333 333 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 189 189 189 333 333 333 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 188 188 188 330 330 330 F F F 17 340-359 186 186 186 328 328 328 F F F 18 360-379 187 187 187 329 329 329 F F F 19 380-399 188 188 188 332 332 332 F F F 20 400-419 188 188 188 330 330 330 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 186 186 186 327 327 327 F F F 24 480-499 189 189 189 333 333 333 F F F 25 500-519 189 189 189 333 333 333 F F F 26 520-539 175 175 175 306 306 306 F F F 27 540-559 184 184 184 324 324 324 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 181 181 181 315 315 315 F F F Max gen: 189, Max flush: 333 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.028461477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.072361647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073000958Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073044279Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073060155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073074642Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} {"msg":"Crucible f2a6906d-852e-4c54-bfb8-e6f9bccddd88 has session id: 6d5c536b-a553-459e-a265-bb2ab9d3f7df","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073645418Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Upstairs opts: Upstairs UUID: f2a6906d-852e-4c54-bfb8-e6f9bccddd88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073690139Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Crucible stats registered with UUID: f2a6906d-852e-4c54-bfb8-e6f9bccddd88","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.073711644Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:25.07373234Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074165176Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074189191Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} {"msg":"f2a6906d-852e-4c54-bfb8-e6f9bccddd88 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074252017Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074268963Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074292798Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074310194Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074324671Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074338588Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074352815Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074365502Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074382229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074407233Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"ds_connection connected from Ok(127.0.0.1:63410)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074433678Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"ds_connection connected from Ok(127.0.0.1:34742)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074761767Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"ds_connection connected from Ok(127.0.0.1:48475)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.074801118Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075129027Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075157771Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075364026Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075845572Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075891052Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.075962927Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076419008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.07645267Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 330, 315, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076477025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[0]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 188, 181, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076502719Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076527094Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 330, 315, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076551539Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[1]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 188, 181, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076577713Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076603278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 330, 315, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076627883Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[2]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 188, 181, 188, 189, 183]","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076662215Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076692058Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076722882Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076738818Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076752245Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076765183Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"":"downstairs","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076788957Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076801285Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"0","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076814232Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"1","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076827069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"client":"2","":"downstairs","session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076840146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076852054Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"f2a6906d-852e-4c54-bfb8-e6f9bccddd88 is now active with session: 6d5c536b-a553-459e-a265-bb2ab9d3f7df","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.07687223Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-23T17:46:25.076889266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944,"session_id":"6d5c536b-a553-459e-a265-bb2ab9d3f7df"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:25.076904992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 24576 data: 12 12 13 12 13 14 02/30 Write block 491 len 20480 data: 13 12 8 11 9 03/30 Read block 390 len 28672 04/30 Read block 476 len 4096 05/30 Read block 30 len 40960 06/30 Write block 392 len 4096 data: 17 07/30 Read block 566 len 24576 08/30 Write block 544 len 4096 data: 23 09/30 Write block 525 len 36864 data: 16 16 11 11 15 14 18 20 21 10/30 Flush 11/30 Write block 594 len 24576 data: 8 8 6 6 5 2 12/30 Flush 13/30 Write block 169 len 4096 data: 14 14/30 Read block 447 len 32768 15/30 Write block 397 len 24576 data: 14 17 18 14 15 14 16/30 Read block 383 len 8192 17/30 Read block 46 len 12288 18/30 Write block 171 len 4096 data: 16 19/30 Read block 34 len 20480 20/30 Write block 460 len 24576 data: 12 12 15 11 13 14 21/30 Flush 22/30 Write block 151 len 28672 data: 11 11 13 14 12 15 17 23/30 Read block 182 len 28672 24/30 Flush 25/30 Write block 333 len 32768 data: 14 13 13 14 11 16 18 19 26/30 Write block 594 len 16384 data: 9 9 7 7 27/30 Write block 353 len 24576 data: 15 14 12 13 12 13 28/30 Read block 158 len 12288 29/30 Read block 311 len 8192 30/30 Read block 97 len 8192 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:25.095262856Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1944} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 189 189 189 333 333 333 F F F 5 100-119 188 188 188 330 330 330 F F F 6 120-139 188 188 188 330 330 330 F F F 7 140-159 190 190 190 337 337 337 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 189 189 189 333 333 333 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 188 188 188 330 330 330 T T T 17 340-359 186 186 186 328 328 328 T T T 18 360-379 187 187 187 329 329 329 F F F 19 380-399 190 190 190 336 336 336 F F F 20 400-419 190 190 190 336 336 336 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 190 190 190 336 336 336 F F F 24 480-499 190 190 190 334 334 334 F F F 25 500-519 189 189 189 333 333 333 F F F 26 520-539 190 190 190 334 334 334 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 190 190 190 335 335 335 T T T Max gen: 190, Max flush: 337 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 189 189 189 333 333 333 F F F 5 100-119 188 188 188 330 330 330 F F F 6 120-139 188 188 188 330 330 330 F F F 7 140-159 190 190 190 337 337 337 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 189 189 189 333 333 333 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 184 184 184 322 322 322 F F F 16 320-339 191 191 191 338 338 338 F F F 17 340-359 191 191 191 338 338 338 F F F 18 360-379 187 187 187 329 329 329 F F F 19 380-399 190 190 190 336 336 336 F F F 20 400-419 190 190 190 336 336 336 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 190 190 190 336 336 336 F F F 24 480-499 190 190 190 334 334 334 F F F 25 500-519 189 189 189 333 333 333 F F F 26 520-539 190 190 190 334 334 334 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 191 191 191 338 338 338 F F F Max gen: 191, Max flush: 338 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.327992013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.372723383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.373556323Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.373608381Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.373627467Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.373644164Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} {"msg":"Crucible 30351657-4de2-4199-9490-2479cb52cecb has session id: b22280be-c19e-497d-b98f-59f9e896f657","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374325706Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Upstairs opts: Upstairs UUID: 30351657-4de2-4199-9490-2479cb52cecb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374371326Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Crucible stats registered with UUID: 30351657-4de2-4199-9490-2479cb52cecb","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.37439867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:29.374428494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374805092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374834406Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} {"msg":"30351657-4de2-4199-9490-2479cb52cecb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374902441Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374919597Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374942023Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374965278Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.374992092Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375017996Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375101508Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375131411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375158625Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375187069Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"ds_connection connected from Ok(127.0.0.1:63648)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375212154Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"ds_connection connected from Ok(127.0.0.1:41811)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375238448Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"ds_connection connected from Ok(127.0.0.1:46744)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375289937Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375858814Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.375901804Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.376064959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.376498515Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.376703411Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.376865616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377102255Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377204932Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 337, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377365667Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[0]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 190, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377392841Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377408528Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 337, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377422665Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[1]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 190, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377436252Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377453758Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 306, 331, 333, 330, 330, 337, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377467495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[2]R generation[0..12]: [184, 184, 175, 188, 189, 188, 188, 190, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377481153Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:46:29.37749513Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377517155Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377542489Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377560055Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377573033Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"":"downstairs","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.37758562Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377597237Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"0","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377609914Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"1","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377622342Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"client":"2","":"downstairs","session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377635049Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377646657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"30351657-4de2-4199-9490-2479cb52cecb is now active with session: b22280be-c19e-497d-b98f-59f9e896f657","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377659024Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"Set Active after no reconciliationDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.37767692Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952,"session_id":"b22280be-c19e-497d-b98f-59f9e896f657"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:29.377689287Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 505 len 4096 data: 16 02/30 Write block 376 len 36864 data: 16 13 10 11 14 14 17 16 18 03/30 Read block 303 len 24576 04/30 Read block 23 len 16384 05/30 Write block 29 len 12288 data: 17 14 13 06/30 Read block 20 len 28672 07/30 Flush 08/30 Write block 394 len 40960 data: 19 17 16 15 18 19 15 16 15 15 09/30 Write block 535 len 4096 data: 17 10/30 Flush 11/30 Write block 464 len 12288 data: 14 15 16 12/30 Read block 22 len 24576 13/30 Flush 14/30 Write block 462 len 40960 data: 16 12 15 16 17 18 18 12 13 14 15/30 Write block 301 len 4096 data: 18 16/30 Read block 217 len 32768 17/30 Flush 18/30 Write block 136 len 28672 data: 15 13 11 13 14 12 13 19/30 Read block 575 len 4096 20/30 Read block 1 len 40960 21/30 Read block 422 len 28672 22/30 Write block 138 len 8192 data: 12 14 23/30 Write block 93 len 28672 data: 12 11 9 7 10 10 8 24/30 Flush 25/30 Write block 317 len 32768 data: 9 8 10 11 12 10 10 11 26/30 Write block 378 len 32768 data: 11 12 15 15 18 17 19 15 27/30 Write block 493 len 20480 data: 9 12 10 12 12 28/30 Read block 429 len 16384 29/30 Write block 587 len 16384 data: 8 10 8 7 30/30 Read block 16 len 12288 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:29.397450387Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1952} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 192 192 192 339 339 339 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 188 188 188 330 330 330 F F F 6 120-139 192 192 192 343 343 343 F F F 7 140-159 192 192 192 343 343 343 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 189 189 189 333 333 333 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 192 192 192 342 342 342 T T T 16 320-339 191 191 191 338 338 338 T T T 17 340-359 191 191 191 338 338 338 F F F 18 360-379 192 192 192 339 339 339 T T T 19 380-399 192 192 192 340 340 340 T T T 20 400-419 192 192 192 340 340 340 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 192 192 192 342 342 342 F F F 24 480-499 190 190 190 334 334 334 T T T 25 500-519 192 192 192 339 339 339 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 191 191 191 338 338 338 T T T Max gen: 192, Max flush: 343 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 323 323 323 F F F 1 020-039 192 192 192 339 339 339 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 188 188 188 330 330 330 F F F 6 120-139 192 192 192 343 343 343 F F F 7 140-159 192 192 192 343 343 343 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 189 189 189 333 333 333 F F F 11 220-239 183 183 183 321 321 321 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 193 193 193 344 344 344 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 191 191 191 338 338 338 F F F 18 360-379 193 193 193 344 344 344 F F F 19 380-399 193 193 193 344 344 344 F F F 20 400-419 192 192 192 340 340 340 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 192 192 192 342 342 342 F F F 24 480-499 193 193 193 344 344 344 F F F 25 500-519 192 192 192 339 339 339 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 193 193 193 344 344 344 F F F Max gen: 193, Max flush: 344 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.648270935Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.692350836Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693238904Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693290073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693309059Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693332933Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} {"msg":"Crucible 8f64be0b-a953-48f2-88fd-e8c056401e60 has session id: 5e5764f9-c1dd-444e-983c-0a222f989882","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693923435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Upstairs opts: Upstairs UUID: 8f64be0b-a953-48f2-88fd-e8c056401e60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.693969036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Crucible stats registered with UUID: 8f64be0b-a953-48f2-88fd-e8c056401e60","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69399533Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:33.694018905Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694431495Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.6944573Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} {"msg":"8f64be0b-a953-48f2-88fd-e8c056401e60 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694518477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69454604Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694576404Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694602848Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694631992Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694661366Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694691229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694722073Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694767852Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694799725Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"ds_connection connected from Ok(127.0.0.1:32817)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69482592Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"ds_connection connected from Ok(127.0.0.1:50028)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.694850165Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"ds_connection connected from Ok(127.0.0.1:46613)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69501025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.695547034Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.695575997Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.696233145Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.696274616Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.696943551Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.696968766Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.697072723Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.697630203Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[0]R flush_numbers[0..12]: [323, 339, 306, 331, 343, 330, 343, 343, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698161527Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[0]R generation[0..12]: [184, 192, 175, 188, 192, 188, 192, 192, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698188871Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698205108Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[1]R flush_numbers[0..12]: [323, 339, 306, 331, 343, 330, 343, 343, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698219315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[1]R generation[0..12]: [184, 192, 175, 188, 192, 188, 192, 192, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698233242Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698246879Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[2]R flush_numbers[0..12]: [323, 339, 306, 331, 343, 330, 343, 343, 336, 331, 333, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698260226Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[2]R generation[0..12]: [184, 192, 175, 188, 192, 188, 192, 192, 190, 188, 189, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698273383Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69828698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698299647Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX"Wait for a query_work_queue command to finish before sending IO ,"pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698319793Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.69833243Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698344838Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"":"downstairs","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698357415Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698368903Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"0","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698384549Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"1","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698397716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"client":"2","":"downstairs","session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698410494Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698422331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"8f64be0b-a953-48f2-88fd-e8c056401e60 is now active with session: 5e5764f9-c1dd-444e-983c-0a222f989882","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698434698Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698446136Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960,"session_id":"5e5764f9-c1dd-444e-983c-0a222f989882"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:33.698458013Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 28672 data: 13 16 16 19 18 20 16 02/30 Write block 423 len 40960 data: 14 18 17 18 20 18 16 17 12 12 03/30 Read block 261 len 20480 04/30 Read block 501 len 32768 05/30 Flush 06/30 Write block 3 len 24576 data: 11 12 16 18 16 16 07/30 Read block 319 len 28672 08/30 Read block 69 len 28672 09/30 Write block 492 len 36864 data: 13 10 13 11 13 13 11 11 13 10/30 Read block 498 len 36864 11/30 Write block 226 len 8192 data: 13 13 12/30 Read block 560 len 32768 13/30 Read block 579 len 28672 14/30 Read block 57 len 28672 15/30 Read block 17 len 24576 16/30 Read block 540 len 36864 17/30 Flush 18/30 Write block 494 len 4096 data: 14 19/30 Read block 384 len 4096 20/30 Read block 462 len 16384 21/30 Write block 200 len 16384 data: 10 11 16 15 22/30 Read block 207 len 20480 23/30 Flush 24/30 Write block 212 len 8192 data: 17 12 25/30 Write block 117 len 40960 data: 15 18 21 21 20 20 18 18 19 13 26/30 Read block 256 len 40960 27/30 Write block 21 len 20480 data: 12 12 13 13 13 28/30 Read block 530 len 40960 29/30 Read block 521 len 32768 30/30 Read block 398 len 40960 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:33.720047716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1960} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 346 346 346 F F F 1 020-039 192 192 192 339 339 339 T T T 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 188 188 188 330 330 330 T T T 6 120-139 192 192 192 343 343 343 T T T 7 140-159 192 192 192 343 343 343 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 194 194 194 347 347 347 T T T 11 220-239 194 194 194 346 346 346 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 193 193 193 344 344 344 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 191 191 191 338 338 338 F F F 18 360-379 194 194 194 345 345 345 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 192 192 192 340 340 340 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 192 192 192 342 342 342 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 193 193 193 344 344 344 F F F Max gen: 194, Max flush: 347 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 346 346 346 F F F 1 020-039 195 195 195 348 348 348 F F F 2 040-059 175 175 175 306 306 306 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 195 195 195 348 348 348 F F F 6 120-139 195 195 195 348 348 348 F F F 7 140-159 192 192 192 343 343 343 F F F 8 160-179 190 190 190 336 336 336 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 194 194 194 346 346 346 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 193 193 193 344 344 344 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 191 191 191 338 338 338 F F F 18 360-379 194 194 194 345 345 345 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 192 192 192 340 340 340 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 192 192 192 342 342 342 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 F F F 29 580-599 193 193 193 344 344 344 F F F Max gen: 195, Max flush: 348 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:37.961772244Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.008379727Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009062849Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.00910657Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009121547Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009135613Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} {"msg":"Crucible 874f69ba-909a-47b8-9dd2-f8207a7ab5c5 has session id: eddc28db-6ddf-4f92-9911-dc140d5e45e3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009717167Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Upstairs opts: Upstairs UUID: 874f69ba-909a-47b8-9dd2-f8207a7ab5c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009763648Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Crucible stats registered with UUID: 874f69ba-909a-47b8-9dd2-f8207a7ab5c5","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.009779734Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:38.009792461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010156052Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010180067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} {"msg":"874f69ba-909a-47b8-9dd2-f8207a7ab5c5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010246423Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010263329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010285754Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010314218Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010341042Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010366067Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010393691Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010419945Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010446149Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010503827Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42912)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010531721Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51208)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010554896Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64663)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.010579181Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.011187189Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.01132251Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.011421008Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.011843897Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.011957132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.011988606Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012420132Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012594114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[0]R flush_numbers[0..12]: [346, 348, 306, 331, 343, 348, 348, 343, 336, 331, 348, 346]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012654041Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[0]R generation[0..12]: [194, 195, 175, 188, 192, 195, 195, 192, 190, 188, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012674147Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012688564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[1]R flush_numbers[0..12]: [346, 348, 306, 331, 343, 348, 348, 343, 336, 331, 348, 346]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012702361Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[1]R generation[0..12]: [194, 195, 175, 188, 192, 195, 195, 192, 190, 188, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012715718Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012729025Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[2]R flush_numbers[0..12]: [346, 348, 306, 331, 343, 348, 348, 343, 336, 331, 348, 346]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012742392Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[2]R generation[0..12]: [194, 195, 175, 188, 192, 195, 195, 192, 190, 188, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012756039Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012769486Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:46:38.012782114Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012804529Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012817036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012829773Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"":"downstairs","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.01284263Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012854318Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"0","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012867585Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"1","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012880432Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"client":"2","":"downstairs","session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.01289327Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012905127Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"874f69ba-909a-47b8-9dd2-f8207a7ab5c5 is now active with session: eddc28db-6ddf-4f92-9911-dc140d5e45e3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012917524Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012929142Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968,"session_id":"eddc28db-6ddf-4f92-9911-dc140d5e45e3"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:38.012940959Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 40960 data: 6 8 10 12 13 17 19 17 17 13 02/30 Read block 441 len 20480 03/30 Write block 104 len 16384 data: 12 13 14 13 04/30 Write block 227 len 40960 data: 14 16 16 13 15 18 17 16 19 17 05/30 Read block 103 len 4096 06/30 Write block 191 len 12288 data: 19 23 16 07/30 Read block 467 len 24576 08/30 Read block 487 len 20480 09/30 Read block 470 len 20480 10/30 Read block 127 len 20480 11/30 Flush 12/30 Write block 194 len 20480 data: 13 14 9 9 7 13/30 Read block 253 len 36864 14/30 Read block 193 len 4096 15/30 Read block 470 len 28672 16/30 Read block 230 len 12288 17/30 Read block 46 len 12288 18/30 Read block 498 len 28672 19/30 Write block 33 len 36864 data: 14 16 14 16 14 15 13 12 15 20/30 Read block 417 len 24576 21/30 Write block 173 len 36864 data: 12 13 12 7 9 14 13 15 14 22/30 Write block 310 len 40960 data: 15 13 12 12 10 9 10 10 9 11 23/30 Write block 342 len 36864 data: 18 20 17 20 17 16 12 14 14 24/30 Write block 0 len 32768 data: 7 9 11 13 14 18 20 18 25/30 Write block 177 len 32768 data: 10 15 14 16 15 17 21 21 26/30 Flush 27/30 Write block 561 len 36864 data: 12 12 10 13 17 20 19 14 16 28/30 Read block 46 len 40960 29/30 Write block 23 len 40960 data: 14 14 14 12 14 17 18 15 14 15 30/30 Write block 473 len 12288 data: 11 12 11 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:38.034858461Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1968} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 350 350 350 F F F 1 020-039 196 196 196 350 350 350 T T T 2 040-059 196 196 196 350 350 350 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 196 196 196 349 349 349 F F F 6 120-139 195 195 195 348 348 348 F F F 7 140-159 192 192 192 343 343 343 F F F 8 160-179 196 196 196 350 350 350 F F F 9 180-199 196 196 196 350 350 350 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 196 196 196 350 350 350 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 196 196 196 350 350 350 F F F 18 360-379 194 194 194 345 345 345 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 192 192 192 340 340 340 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 192 192 192 342 342 342 T T T 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 184 184 184 324 324 324 T T T 29 580-599 193 193 193 344 344 344 F F F Max gen: 196, Max flush: 350 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 350 350 350 F F F 1 020-039 197 197 197 351 351 351 F F F 2 040-059 196 196 196 350 350 350 F F F 3 060-079 188 188 188 331 331 331 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 196 196 196 349 349 349 F F F 6 120-139 195 195 195 348 348 348 F F F 7 140-159 192 192 192 343 343 343 F F F 8 160-179 196 196 196 350 350 350 F F F 9 180-199 196 196 196 350 350 350 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 196 196 196 350 350 350 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 196 196 196 350 350 350 F F F 18 360-379 194 194 194 345 345 345 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 192 192 192 340 340 340 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 188 188 188 331 331 331 F F F 23 460-479 197 197 197 351 351 351 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 197 197 197 351 351 351 F F F 29 580-599 193 193 193 344 344 344 F F F Max gen: 197, Max flush: 351 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.26628609Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:42.308274744Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.308692724Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.339743867Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.340429929Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2d334f767f4e70887c1c804fc50babae5e22bede\",\n git_commit_timestamp: \"2025-01-23T17:21:44.000000000Z\",\n git_branch: \"renovate/tokio-util-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.340465661Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.340479778Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.340494315Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} {"msg":"Crucible bc283a24-911f-4948-8faf-de9688a3efd2 has session id: 33727d75-a3b0-469c-9cdd-94ab5b0e3c34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341033068Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Upstairs opts: Upstairs UUID: bc283a24-911f-4948-8faf-de9688a3efd2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341088716Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Crucible stats registered with UUID: bc283a24-911f-4948-8faf-de9688a3efd2","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341111331Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:42.341124998Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341534689Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341558674Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} {"msg":"bc283a24-911f-4948-8faf-de9688a3efd2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341616772Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341643036Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341667271Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341681928Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341696564Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341710151Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341724978Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341738955Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341753262Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341769898Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"ds_connection connected from Ok(127.0.0.1:33306)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341833435Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"ds_connection connected from Ok(127.0.0.1:47176)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.341879035Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"ds_connection connected from Ok(127.0.0.1:62288)","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.342064614Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.342658756Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.342690289Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.342705416Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.343251188Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.343398696Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.343426999Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.34379311Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.343987268Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[0]R flush_numbers[0..12]: [350, 351, 350, 331, 343, 349, 348, 343, 350, 350, 348, 349]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344119329Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[0]R generation[0..12]: [196, 197, 196, 188, 192, 196, 195, 192, 196, 196, 195, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344139855Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344155452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[1]R flush_numbers[0..12]: [350, 351, 350, 331, 343, 349, 348, 343, 350, 350, 348, 349]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344169229Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[1]R generation[0..12]: [196, 197, 196, 188, 192, 196, 195, 192, 196, 196, 195, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344187375Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344202062Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[2]R flush_numbers[0..12]: [350, 351, 350, 331, 343, 349, 348, 343, 350, 350, 348, 349]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344215808Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[2]R generation[0..12]: [196, 197, 196, 188, 192, 196, 195, 192, 196, 196, 195, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344229266Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344248452Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-23T17:46:42.344272146Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344292842Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344305859Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344318386Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"":"downstairs","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344331374Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344343091Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"0","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344356078Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"1","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344368826Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"client":"2","":"downstairs","session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344381883Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344397569Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"bc283a24-911f-4948-8faf-de9688a3efd2 is now active with session: 33727d75-a3b0-469c-9cdd-94ab5b0e3c34","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344410477Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.344422184Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976,"session_id":"33727d75-a3b0-469c-9cdd-94ab5b0e3c34"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-23T17:46:42.34444148Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 459 len 12288 data: 12 13 13 02/30 Read block 348 len 8192 03/30 Read block 9 len 12288 04/30 Write block 137 len 28672 data: 14 13 15 15 13 14 11 05/30 Read block 586 len 20480 06/30 Read block 361 len 24576 07/30 Read block 577 len 40960 08/30 Write block 314 len 36864 data: 11 10 11 11 10 12 12 13 11 09/30 Flush 10/30 Write block 363 len 32768 data: 10 10 10 10 9 10 11 16 11/30 Flush 12/30 Write block 305 len 8192 data: 16 15 13/30 Read block 146 len 12288 14/30 Write block 506 len 12288 data: 16 14 13 15/30 Write block 100 len 36864 data: 11 12 12 14 13 14 15 14 13 16/30 Write block 410 len 40960 data: 19 21 21 14 17 16 13 15 17 13 17/30 Write block 580 len 12288 data: 14 15 14 18/30 Write block 414 len 8192 data: 18 17 19/30 Write block 70 len 32768 data: 13 14 15 17 13 12 10 9 20/30 Read block 50 len 32768 21/30 Write block 329 len 28672 data: 14 17 15 16 15 14 14 22/30 Read block 85 len 20480 23/30 Read block 214 len 36864 24/30 Flush 25/30 Flush 26/30 Write block 162 len 24576 data: 15 14 15 10 12 11 27/30 Read block 568 len 4096 28/30 Read block 152 len 32768 29/30 Read block 93 len 4096 30/30 Write block 489 len 24576 data: 19 16 14 14 11 15 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-23T17:46:42.364675427Z","hostname":"w-01JJA3S20ESG10RT34W8GJ5BPX","pid":1976} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 350 350 350 F F F 1 020-039 197 197 197 351 351 351 F F F 2 040-059 196 196 196 350 350 350 F F F 3 060-079 198 198 198 354 354 354 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 198 198 198 354 354 354 F F F 6 120-139 198 198 198 352 352 352 F F F 7 140-159 198 198 198 352 352 352 F F F 8 160-179 196 196 196 350 350 350 T T T 9 180-199 196 196 196 350 350 350 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 198 198 198 354 354 354 F F F 16 320-339 198 198 198 354 354 354 F F F 17 340-359 196 196 196 350 350 350 F F F 18 360-379 198 198 198 353 353 353 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 198 198 198 354 354 354 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 198 198 198 352 352 352 F F F 23 460-479 198 198 198 352 352 352 F F F 24 480-499 194 194 194 347 347 347 T T T 25 500-519 198 198 198 354 354 354 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 197 197 197 351 351 351 F F F 29 580-599 198 198 198 354 354 354 F F F Max gen: 198, Max flush: 354 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 350 350 350 F F F 1 020-039 197 197 197 351 351 351 F F F 2 040-059 196 196 196 350 350 350 F F F 3 060-079 198 198 198 354 354 354 F F F 4 080-099 192 192 192 343 343 343 F F F 5 100-119 198 198 198 354 354 354 F F F 6 120-139 198 198 198 352 352 352 F F F 7 140-159 198 198 198 352 352 352 F F F 8 160-179 199 199 199 355 355 355 F F F 9 180-199 196 196 196 350 350 350 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 190 190 190 334 334 334 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 198 198 198 354 354 354 F F F 16 320-339 198 198 198 354 354 354 F F F 17 340-359 196 196 196 350 350 350 F F F 18 360-379 198 198 198 353 353 353 F F F 19 380-399 194 194 194 345 345 345 F F F 20 400-419 198 198 198 354 354 354 F F F 21 420-439 194 194 194 345 345 345 F F F 22 440-459 198 198 198 352 352 352 F F F 23 460-479 198 198 198 352 352 352 F F F 24 480-499 199 199 199 355 355 355 F F F 25 500-519 198 198 198 354 354 354 F F F 26 520-539 192 192 192 340 340 340 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 197 197 197 351 351 351 F F F 29 580-599 198 198 198 354 354 354 F F F Max gen: 199, Max flush: 355 9:15 Test duration Test completed real 9:14.789436586 user 49.818993476 sys 19.481173028 trap 0.072555528 tflt 0.084025238 dflt 0.132698989 kflt 0.008160918 lock 14:52:00.316934793 slp 45:22.114088112 lat 5.757131312 stop 0.066562249