Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.268373 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.019105386 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020102762 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.748327125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.813437303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.814901921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.81498864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.815023726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.815040814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} {"msg":"Crucible 00aec9cc-0e81-46a7-9768-df4af3e22f8d has session id: 97f1efde-4c40-4809-9c4e-6781f1370973","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.815915185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Upstairs opts: Upstairs UUID: 00aec9cc-0e81-46a7-9768-df4af3e22f8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.815970278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Crucible stats registered with UUID: 00aec9cc-0e81-46a7-9768-df4af3e22f8d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.816013503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:22.816035101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817136544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817196836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} {"msg":"00aec9cc-0e81-46a7-9768-df4af3e22f8d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817227722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.81724448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817309752Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.81732769Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817341698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817355456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817369245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817384093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817410539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817438186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_connection connected from Ok(127.0.0.1:38570)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817673607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_connection connected from Ok(127.0.0.1:51035)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.817754897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_connection connected from Ok(127.0.0.1:60593)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.81789047Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819477813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819510569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819538415Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819633063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819670569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.819687707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82070177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.820744425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.820771332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821554005Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82159335Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821619796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821642114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821666381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821691048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821713835Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821738282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821797014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82183024Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821847168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82191141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821932158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821947306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821961474Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821975262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.821989641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822011738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822031345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822054082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822133163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"":"downstairs","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82215236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822165179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"0","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822179067Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"1","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822196515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"client":"2","":"downstairs","session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822210563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822223361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"00aec9cc-0e81-46a7-9768-df4af3e22f8d is now active with session: 97f1efde-4c40-4809-9c4e-6781f1370973","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.82223655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822248888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188,"session_id":"97f1efde-4c40-4809-9c4e-6781f1370973"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:22.822261387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:26.916104935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1188} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.946802211Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.99114603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992095322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992131638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992151185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992165873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} {"msg":"Crucible 1b954b30-038b-4fc5-ba14-3b37eb1c30e9 has session id: 8445d20e-7743-4e33-877f-8f651639894f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992626106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Upstairs opts: Upstairs UUID: 1b954b30-038b-4fc5-ba14-3b37eb1c30e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992670331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Crucible stats registered with UUID: 1b954b30-038b-4fc5-ba14-3b37eb1c30e9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.992697187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:26.992719705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993261547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993286994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} {"msg":"1b954b30-038b-4fc5-ba14-3b37eb1c30e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993355026Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993374523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.99339964Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993426347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993453223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.99347978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993513036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993542992Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993574778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993603045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34291)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993629621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58352)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993746147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34559)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.993808329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994424673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994454109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994470357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994484675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994788317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.994817584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995095619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995285426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995448475Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995685716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995712562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995727531Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995786303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995807041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995820839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995960672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995979199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.995995567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996021914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996036922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996066338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996081037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996095365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996112653Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996126411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996140249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996153198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996166356Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996179424Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996192393Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"":"downstairs","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:40:26.996205831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996223679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"0","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996236917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"1","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996249916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"client":"2","":"downstairs","session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996262994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996275383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"1b954b30-038b-4fc5-ba14-3b37eb1c30e9 is now active with session: 8445d20e-7743-4e33-877f-8f651639894f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996288221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996300349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190,"session_id":"8445d20e-7743-4e33-877f-8f651639894f"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:26.996312718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 440 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 472 len 8192 data: 2 2 04/30 Read block 171 len 28672 05/30 Read block 429 len 24576 06/30 Read block 395 len 36864 07/30 Flush 08/30 Write block 391 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Write block 29 len 36864 data: 2 2 2 2 2 2 2 2 2 10/30 Flush 11/30 Write block 29 len 12288 data: 3 3 3 12/30 Write block 513 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Flush 14/30 Write block 200 len 4096 data: 2 15/30 Read block 184 len 8192 16/30 Write block 176 len 12288 data: 2 2 2 17/30 Read block 499 len 4096 18/30 Read block 547 len 20480 19/30 Write block 320 len 4096 data: 2 20/30 Write block 211 len 24576 data: 2 2 2 2 2 2 21/30 Flush 22/30 Flush 23/30 Write block 231 len 16384 data: 2 2 2 2 24/30 Flush 25/30 Write block 66 len 24576 data: 2 2 2 2 2 2 26/30 Read block 307 len 16384 27/30 Write block 517 len 12288 data: 3 3 3 28/30 Write block 234 len 8192 data: 3 2 29/30 Write block 299 len 16384 data: 2 2 2 2 30/30 Read block 170 len 20480 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 6 6 4 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:27.014651399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 6 6 6 F F F 11 220-239 2 2 2 8 8 8 T T T 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 2 2 2 6 6 6 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 5 5 5 T T T 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 8 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 9 9 9 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 6 6 6 F F F 11 220-239 3 3 3 9 9 9 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 3 3 3 9 9 9 F F F 15 300-319 3 3 3 9 9 9 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 9 9 9 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 9 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.279690262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32388803Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.324588393Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.324624008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.324638867Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.324653905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} {"msg":"Crucible 9fe4716b-7c23-4b3e-a04b-0752f27f67ef has session id: 7e04d3cc-72e2-4ded-ba06-431360e7d88b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325112008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Upstairs opts: Upstairs UUID: 9fe4716b-7c23-4b3e-a04b-0752f27f67ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325157972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Crucible stats registered with UUID: 9fe4716b-7c23-4b3e-a04b-0752f27f67ef","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325185329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:31.325207486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32581928Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325846296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} {"msg":"9fe4716b-7c23-4b3e-a04b-0752f27f67ef active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32590354Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325922767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325947244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.325961762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32598064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326005237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326031813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32606241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326084507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326100605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64516)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326114763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65504)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326149659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50831)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326349154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326841293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326871399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326886747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.326900876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32702788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.327076984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.327551605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.327581121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.327632705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328179926Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328219822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328245258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328260767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328279414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328293632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328306071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328319469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[0]R flush_numbers[0..12]: [1, 5, 1, 9, 1, 1, 1, 1, 6, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328333138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[0]R generation[0..12]: [1, 2, 1, 3, 1, 1, 1, 1, 2, 1, 2, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328346896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328361124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[1]R flush_numbers[0..12]: [1, 5, 1, 9, 1, 1, 1, 1, 6, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328374762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[1]R generation[0..12]: [1, 2, 1, 3, 1, 1, 1, 1, 2, 1, 2, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328388491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328406169Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[2]R flush_numbers[0..12]: [1, 5, 1, 9, 1, 1, 1, 1, 6, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328431745Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[2]R generation[0..12]: [1, 2, 1, 3, 1, 1, 1, 1, 2, 1, 2, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328465661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328486099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328500467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328517275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328531523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328544701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"":"downstairs","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32855831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328570698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"0","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328584156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"1","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328597345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"client":"2","":"downstairs","session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328610733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328623291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"9fe4716b-7c23-4b3e-a04b-0752f27f67ef is now active with session: 7e04d3cc-72e2-4ded-ba06-431360e7d88b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.32863629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328648418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198,"session_id":"7e04d3cc-72e2-4ded-ba06-431360e7d88b"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:31.328660847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 181 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 348 len 24576 data: 2 2 2 2 2 2 03/30 Read block 211 len 32768 04/30 Write block 57 len 24576 data: 2 2 2 2 2 2 05/30 Read block 487 len 40960 06/30 Read block 273 len 12288 07/30 Write block 593 len 12288 data: 2 2 2 08/30 Read block 505 len 28672 09/30 Read block 103 len 24576 10/30 Write block 193 len 40960 data: 2 2 2 2 2 2 2 3 2 2 11/30 Read block 94 len 12288 12/30 Read block 484 len 24576 13/30 Read block 73 len 20480 14/30 Write block 256 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Write block 201 len 20480 data: 3 3 2 2 2 16/30 Write block 181 len 36864 data: 3 3 3 3 3 3 3 3 2 17/30 Write block 24 len 8192 data: 2 2 18/30 Read block 433 len 16384 19/30 Flush 20/30 Write block 193 len 24576 data: 3 3 3 3 3 3 21/30 Read block 468 len 40960 22/30 Read block 580 len 28672 23/30 Write block 313 len 4096 data: 2 24/30 Flush 25/30 Write block 299 len 12288 data: 3 3 3 26/30 Read block 422 len 8192 27/30 Read block 379 len 40960 28/30 Write block 499 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Read block 295 len 16384 30/30 Write block 122 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:31.348849068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 4 4 4 10 10 10 F F F 2 040-059 4 4 4 10 10 10 F F F 3 060-079 4 4 4 10 10 10 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 4 4 4 11 11 11 F F F 10 200-219 4 4 4 10 10 10 F F F 11 220-239 3 3 3 9 9 9 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 4 4 4 10 10 10 F F F 14 280-299 3 3 3 9 9 9 T T T 15 300-319 4 4 4 11 11 11 T T T 16 320-339 2 2 2 6 6 6 F F F 17 340-359 4 4 4 10 10 10 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 3 3 3 9 9 9 T T T 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 10 10 10 F F F Max gen: 4, Max flush: 11 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 4 4 4 10 10 10 F F F 2 040-059 4 4 4 10 10 10 F F F 3 060-079 4 4 4 10 10 10 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 12 12 12 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 4 4 4 11 11 11 F F F 10 200-219 4 4 4 10 10 10 F F F 11 220-239 3 3 3 9 9 9 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 4 4 4 10 10 10 F F F 14 280-299 5 5 5 12 12 12 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 4 4 4 10 10 10 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 5 5 5 12 12 12 F F F 25 500-519 5 5 5 12 12 12 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 10 10 10 F F F Max gen: 5, Max flush: 12 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.58606483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:35.628077109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.628441493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.658977069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.659555387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.659590463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.659605051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.659620569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} {"msg":"Crucible 8a5ca9cc-655c-4b4f-9f13-4f0a6ebd6ec5 has session id: 99508b40-18de-4c5c-8f44-940c9fd1a1b6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660103589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Upstairs opts: Upstairs UUID: 8a5ca9cc-655c-4b4f-9f13-4f0a6ebd6ec5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660148713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Crucible stats registered with UUID: 8a5ca9cc-655c-4b4f-9f13-4f0a6ebd6ec5","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660167981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:35.660185829Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660567131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660591628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} {"msg":"8a5ca9cc-655c-4b4f-9f13-4f0a6ebd6ec5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660651341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660670099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660692916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660708264Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660723202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660744309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660773256Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660844737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660888131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660918288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33851)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660948154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33828)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.660974441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33417)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661001957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661582835Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661611671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661626889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661642168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661656246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.661672154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662291667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662334332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662465185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662843608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662872195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662887733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662956454Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.662977022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.66299135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663023056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663038034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[0]R flush_numbers[0..12]: [1, 10, 10, 10, 1, 1, 12, 1, 6, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663052132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[0]R generation[0..12]: [1, 4, 4, 4, 1, 1, 5, 1, 2, 4, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663066181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663080559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[1]R flush_numbers[0..12]: [1, 10, 10, 10, 1, 1, 12, 1, 6, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663094367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[1]R generation[0..12]: [1, 4, 4, 4, 1, 1, 5, 1, 2, 4, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663107976Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663122014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[2]R flush_numbers[0..12]: [1, 10, 10, 10, 1, 1, 12, 1, 6, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663135832Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[2]R generation[0..12]: [1, 4, 4, 4, 1, 1, 5, 1, 2, 4, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.66314966Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663164198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663177677Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663191295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663204633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663217972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663241819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663254097Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663268006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"1","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663281694Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"client":"2","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663295412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663308001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"8a5ca9cc-655c-4b4f-9f13-4f0a6ebd6ec5 is now active with session: 99508b40-18de-4c5c-8f44-940c9fd1a1b6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663321149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663333568Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:35.663345956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 447 len 32768 data: 3 2 2 2 2 2 2 2 02/30 Read block 221 len 12288 03/30 Read block 101 len 28672 04/30 Write block 347 len 20480 data: 2 3 3 3 3 05/30 Write block 568 len 8192 data: 2 2 06/30 Write block 232 len 4096 data: 3 07/30 Flush 08/30 Write block 153 len 24576 data: 2 2 2 2 2 2 09/30 Read block 255 len 40960 10/30 Read block 507 len 32768 11/30 Flush 12/30 Write block 54 len 36864 data: 2 2 2 3 3 3 3 3 3 13/30 Flush 14/30 Write block 570 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 62 len 20480 16/30 Flush 17/30 Write block 201 len 24576 data: 4 4 3 3 3 2 18/30 Write block 483 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Write block 68 len 16384 data: 3 3 3 3 20/30 Read block 419 len 40960 21/30 Read block 64 len 24576 22/30 Read block 133 len 16384 23/30 Write block 420 len 20480 data: 2 2 2 2 2 24/30 Read block 428 len 8192 25/30 Flush 26/30 Write block 585 len 12288 data: 2 2 2 27/30 Read block 411 len 28672 28/30 Write block 12 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Write block 263 len 20480 data: 3 3 3 2 2 30/30 Write block 568 len 28672 data: 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:35.682710409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206} Wrote out file "/tmp/test_repair_verify.data" {"msg":"client response queue closed unexpectedly: channel closed; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:35.683206188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"99508b40-18de-4c5c-8f44-940c9fd1a1b6"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 4 4 4 10 10 10 F F F 2 040-059 6 6 6 15 15 15 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 12 12 12 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 4 4 4 11 11 11 F F F 10 200-219 6 6 6 17 17 17 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 4 4 4 10 10 10 T T T 14 280-299 5 5 5 12 12 12 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 6 6 6 17 17 17 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 6 6 6 17 17 17 F F F 25 500-519 5 5 5 12 12 12 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 6 6 6 16 16 16 T T T 29 580-599 4 4 4 10 10 10 T T T Max gen: 6, Max flush: 17 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 18 18 18 F F F 1 020-039 4 4 4 10 10 10 F F F 2 040-059 6 6 6 15 15 15 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 12 12 12 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 2 2 2 6 6 6 F F F 9 180-199 4 4 4 11 11 11 F F F 10 200-219 6 6 6 17 17 17 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 7 7 7 18 18 18 F F F 14 280-299 5 5 5 12 12 12 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 6 6 6 17 17 17 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 6 6 6 17 17 17 F F F 25 500-519 5 5 5 12 12 12 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 7 7 7 18 18 18 F F F 29 580-599 7 7 7 18 18 18 F F F Max gen: 7, Max flush: 18 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:39.940460908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:39.983383194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:39.983751069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.015760791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016424488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016460114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016473852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016488341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} {"msg":"Crucible 765f1a5e-5c19-4d6c-bc53-078e390438ed has session id: de7c4790-3a03-4284-99ce-a3f3fec9ad08","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01689708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Upstairs opts: Upstairs UUID: 765f1a5e-5c19-4d6c-bc53-078e390438ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016940004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Crucible stats registered with UUID: 765f1a5e-5c19-4d6c-bc53-078e390438ed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.016962902Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:40.01697693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01745349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017489726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} {"msg":"765f1a5e-5c19-4d6c-bc53-078e390438ed active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017547629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017576535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017601522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01761686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017631039Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017655316Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017679142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01770275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017727787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017753753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_connection connected from Ok(127.0.0.1:48835)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01778094Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_connection connected from Ok(127.0.0.1:41961)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.017808336Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_connection connected from Ok(127.0.0.1:59137)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.01794135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018562643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018594869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018611237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018627215Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018753749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.018798103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019245468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019275434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019290952Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019809908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019837504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019853092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019932102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019966168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019982786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.019995825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020013332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[0]R flush_numbers[0..12]: [18, 10, 15, 17, 1, 1, 12, 14, 6, 11, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.02003313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[0]R generation[0..12]: [7, 4, 6, 6, 1, 1, 5, 6, 2, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020047558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020061886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[1]R flush_numbers[0..12]: [18, 10, 15, 17, 1, 1, 12, 14, 6, 11, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020075795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[1]R generation[0..12]: [7, 4, 6, 6, 1, 1, 5, 6, 2, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020089643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020104011Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[2]R flush_numbers[0..12]: [18, 10, 15, 17, 1, 1, 12, 14, 6, 11, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.02011755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[2]R generation[0..12]: [7, 4, 6, 6, 1, 1, 5, 6, 2, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020130978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020145136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020158484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020171853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020184911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.02019802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"":"downstairs","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020211388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214Wait for a query_work_queue command to finish before sending IO ,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020235245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"0","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020248933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"1","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020262312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"client":"2","":"downstairs","session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.02027592Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020288458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"765f1a5e-5c19-4d6c-bc53-078e390438ed is now active with session: de7c4790-3a03-4284-99ce-a3f3fec9ad08","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020301547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020313775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214,"session_id":"de7c4790-3a03-4284-99ce-a3f3fec9ad08"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:40.020326144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 261 len 36864 data: 3 3 4 4 4 3 3 2 2 03/30 Read block 321 len 16384 04/30 Read block 501 len 8192 05/30 Flush 06/30 Write block 37 len 16384 data: 3 2 2 2 07/30 Flush 08/30 Write block 180 len 12288 data: 2 4 4 09/30 Read block 146 len 28672 10/30 Read block 146 len 36864 11/30 Read block 110 len 8192 12/30 Write block 399 len 4096 data: 3 13/30 Read block 17 len 16384 14/30 Write block 438 len 12288 data: 2 2 3 15/30 Write block 561 len 16384 data: 2 2 2 2 16/30 Write block 377 len 4096 data: 2 17/30 Read block 150 len 28672 18/30 Read block 187 len 20480 19/30 Write block 130 len 20480 data: 2 2 2 2 2 20/30 Read block 237 len 4096 21/30 Read block 486 len 12288 22/30 Read block 29 len 16384 23/30 Write block 495 len 24576 data: 2 2 2 2 3 3 24/30 Read block 297 len 4096 25/30 Read block 143 len 4096 26/30 Read block 230 len 28672 27/30 Write block 515 len 12288 data: 3 3 4 28/30 Read block 317 len 20480 29/30 Read block 553 len 24576 30/30 Read block 405 len 12288 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 23 22 23 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:40.039113989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 18 18 18 F F F 1 020-039 8 8 8 20 20 20 F F F 2 040-059 8 8 8 20 20 20 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 12 12 12 T T T 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 4 4 4 11 11 11 T T T 10 200-219 6 6 6 17 17 17 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 8 8 8 19 19 19 F F F 14 280-299 5 5 5 12 12 12 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 2 2 2 4 4 4 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 6 6 6 17 17 17 T T T 22 440-459 6 6 6 13 13 13 T T T 23 460-479 2 2 2 3 3 3 F F F 24 480-499 6 6 6 17 17 17 T T T 25 500-519 5 5 5 12 12 12 T T T 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 7 7 7 18 18 18 T T T 29 580-599 7 7 7 18 18 18 F F F Max gen: 8, Max flush: 20 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 18 18 18 F F F 1 020-039 8 8 8 20 20 20 F F F 2 040-059 8 8 8 20 20 20 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 9 9 9 21 21 21 F F F 10 200-219 6 6 6 17 17 17 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 8 8 8 19 19 19 F F F 14 280-299 5 5 5 12 12 12 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 9 9 9 21 21 21 F F F 19 380-399 9 9 9 21 21 21 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 9 9 9 21 21 21 F F F 22 440-459 9 9 9 21 21 21 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 9 9 9 21 21 21 F F F 25 500-519 9 9 9 21 21 21 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 9 9 9 21 21 21 F F F 29 580-599 7 7 7 18 18 18 F F F Max gen: 9, Max flush: 21 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.326608942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.370062792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.370696063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.370733269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.370747857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.370762885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} {"msg":"Crucible 1b85506f-ca8d-4207-a3e4-828d76513060 has session id: fdd8df60-a494-4053-aa23-3446454c7dca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371242955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Upstairs opts: Upstairs UUID: 1b85506f-ca8d-4207-a3e4-828d76513060, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37128539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Crucible stats registered with UUID: 1b85506f-ca8d-4207-a3e4-828d76513060","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371310137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:44.371332574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371789057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371817804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} {"msg":"1b85506f-ca8d-4207-a3e4-828d76513060 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371859929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371876197Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371898814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371913472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37193062Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371956237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.371983723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37200984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372036657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372102129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:35026)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372130255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:43011)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372176579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:62084)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372287826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372857345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372883941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372903029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.372916817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.373508144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37370009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374042537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374084072Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374113688Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374174361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374206517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374234813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37426185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374290706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.374738611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375211392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375240179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[0]R flush_numbers[0..12]: [18, 20, 20, 17, 1, 1, 21, 14, 19, 21, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375259086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[0]R generation[0..12]: [7, 8, 8, 6, 1, 1, 9, 6, 8, 9, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375273914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375288633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[1]R flush_numbers[0..12]: [18, 20, 20, 17, 1, 1, 21, 14, 19, 21, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375302831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[1]R generation[0..12]: [7, 8, 8, 6, 1, 1, 9, 6, 8, 9, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375316589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375331697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[2]R flush_numbers[0..12]: [18, 20, 20, 17, 1, 1, 21, 14, 19, 21, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375345426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[2]R generation[0..12]: [7, 8, 8, 6, 1, 1, 9, 6, 8, 9, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375359034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375372942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375385981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375399329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375412317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375425296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"":"downstairs","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375438364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:40:44.375450542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"0","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37547282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"1","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375486318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"client":"2","":"downstairs","session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375499566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375511975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"1b85506f-ca8d-4207-a3e4-828d76513060 is now active with session: fdd8df60-a494-4053-aa23-3446454c7dca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375524963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.375537012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222,"session_id":"fdd8df60-a494-4053-aa23-3446454c7dca"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:44.37554954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 12288 data: 3 3 3 02/30 Read block 472 len 8192 03/30 Read block 409 len 36864 04/30 Read block 22 len 16384 05/30 Write block 586 len 16384 data: 3 3 2 2 06/30 Write block 502 len 28672 data: 3 3 3 3 3 2 2 07/30 Read block 554 len 12288 08/30 Write block 18 len 24576 data: 3 3 2 2 2 2 09/30 Flush 10/30 Flush 11/30 Write block 288 len 16384 data: 2 2 2 2 12/30 Read block 514 len 20480 13/30 Write block 275 len 12288 data: 2 2 2 14/30 Read block 185 len 16384 15/30 Read block 589 len 40960 16/30 Write block 401 len 28672 data: 2 2 2 2 2 2 2 17/30 Flush 18/30 Write block 397 len 32768 data: 3 3 4 2 3 3 3 3 19/30 Read block 56 len 4096 20/30 Read block 90 len 36864 21/30 Write block 477 len 28672 data: 2 2 2 2 2 2 3 22/30 Flush 23/30 Write block 27 len 40960 data: 2 2 4 4 4 3 3 3 3 3 24/30 Read block 292 len 20480 25/30 Read block 220 len 24576 26/30 Write block 236 len 4096 data: 2 27/30 Read block 262 len 32768 28/30 Write block 25 len 40960 data: 3 2 3 3 5 5 5 4 4 4 29/30 Read block 411 len 40960 30/30 Read block 558 len 12288 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:44.39639106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 22 22 22 F F F 1 020-039 10 10 10 22 22 22 T T T 2 040-059 8 8 8 20 20 20 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 9 9 9 21 21 21 F F F 10 200-219 6 6 6 17 17 17 F F F 11 220-239 6 6 6 13 13 13 T T T 12 240-259 4 4 4 10 10 10 F F F 13 260-279 10 10 10 24 24 24 F F F 14 280-299 10 10 10 24 24 24 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 9 9 9 21 21 21 F F F 19 380-399 10 10 10 25 25 25 F F F 20 400-419 10 10 10 25 25 25 F F F 21 420-439 10 10 10 22 22 22 F F F 22 440-459 9 9 9 21 21 21 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 10 10 10 25 25 25 F F F 25 500-519 10 10 10 22 22 22 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 9 9 9 21 21 21 F F F 29 580-599 10 10 10 22 22 22 F F F Max gen: 10, Max flush: 25 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 22 22 22 F F F 1 020-039 11 11 11 26 26 26 F F F 2 040-059 8 8 8 20 20 20 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 9 9 9 21 21 21 F F F 10 200-219 6 6 6 17 17 17 F F F 11 220-239 11 11 11 26 26 26 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 10 10 10 24 24 24 F F F 14 280-299 10 10 10 24 24 24 F F F 15 300-319 5 5 5 12 12 12 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 9 9 9 21 21 21 F F F 19 380-399 10 10 10 25 25 25 F F F 20 400-419 10 10 10 25 25 25 F F F 21 420-439 10 10 10 22 22 22 F F F 22 440-459 9 9 9 21 21 21 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 10 10 10 25 25 25 F F F 25 500-519 10 10 10 22 22 22 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 9 9 9 21 21 21 F F F 29 580-599 10 10 10 22 22 22 F F F Max gen: 11, Max flush: 26 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.617423833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:48.659874088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.660263529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.692110122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.692760191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.692798136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.692812384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.692826812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} {"msg":"Crucible 9abad530-bb3e-432b-83b5-10690bec1f7c has session id: 8ec944c8-82c9-4e53-96d3-b27d5647e1d3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693306593Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Upstairs opts: Upstairs UUID: 9abad530-bb3e-432b-83b5-10690bec1f7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693335249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Crucible stats registered with UUID: 9abad530-bb3e-432b-83b5-10690bec1f7c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693350377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:48.693363336Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693786443Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69381155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} {"msg":"9abad530-bb3e-432b-83b5-10690bec1f7c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693878972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693902359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693925836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693941124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693954922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69396848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693982389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.693995657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694035882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69405008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62332)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694163096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40729)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694284281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56714)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694320367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694863589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694901655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.694931311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69501588Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.695050686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.695081922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.695547454Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.695649142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.695672009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696153949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696182625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696199863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696260496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696296311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696312079Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696325268Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696339146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[0]R flush_numbers[0..12]: [22, 26, 20, 17, 1, 1, 21, 14, 19, 21, 17, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696352844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[0]R generation[0..12]: [10, 11, 8, 6, 1, 1, 9, 6, 8, 9, 6, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696366653Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69638498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[1]R flush_numbers[0..12]: [22, 26, 20, 17, 1, 1, 21, 14, 19, 21, 17, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696399378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[1]R generation[0..12]: [10, 11, 8, 6, 1, 1, 9, 6, 8, 9, 6, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696413227Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696427295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[2]R flush_numbers[0..12]: [22, 26, 20, 17, 1, 1, 21, 14, 19, 21, 17, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696440883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[2]R generation[0..12]: [10, 11, 8, 6, 1, 1, 9, 6, 8, 9, 6, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696454392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.69646847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696481908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696495777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696509085Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696522323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"":"downstairs","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696535632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-11T23:40:48.69654787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"0","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696569047Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"1","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696585195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"client":"2","":"downstairs","session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696599024Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696611612Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"9abad530-bb3e-432b-83b5-10690bec1f7c is now active with session: 8ec944c8-82c9-4e53-96d3-b27d5647e1d3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696624561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696636779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230,"session_id":"8ec944c8-82c9-4e53-96d3-b27d5647e1d3"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:48.696649247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 8192 data: 3 3 02/30 Flush 03/30 Write block 210 len 20480 data: 2 3 3 3 3 04/30 Read block 308 len 28672 05/30 Write block 189 len 24576 data: 3 2 2 2 4 4 06/30 Write block 435 len 40960 data: 2 2 2 3 3 4 3 3 3 3 07/30 Read block 396 len 16384 08/30 Write block 369 len 20480 data: 2 2 2 2 2 09/30 Write block 361 len 16384 data: 2 2 2 2 10/30 Write block 5 len 40960 data: 2 2 2 2 2 2 2 3 3 3 11/30 Read block 185 len 36864 12/30 Write block 557 len 40960 data: 2 2 2 2 3 3 3 3 2 2 13/30 Write block 454 len 16384 data: 3 2 2 2 14/30 Write block 592 len 32768 data: 2 3 3 3 2 2 2 2 15/30 Write block 302 len 40960 data: 3 2 2 2 2 2 2 2 2 2 16/30 Read block 462 len 4096 17/30 Read block 433 len 24576 18/30 Read block 413 len 32768 19/30 Write block 19 len 40960 data: 4 3 3 3 3 3 4 3 4 4 20/30 Read block 573 len 36864 21/30 Write block 436 len 4096 data: 3 22/30 Write block 483 len 4096 data: 4 23/30 Write block 59 len 4096 data: 4 24/30 Read block 264 len 16384 25/30 Write block 442 len 20480 data: 4 4 4 3 3 26/30 Write block 20 len 40960 data: 4 4 4 4 4 5 4 5 5 6 27/30 Read block 200 len 36864 28/30 Read block 332 len 36864 29/30 Write block 564 len 4096 data: 4 30/30 Write block 552 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 5 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Sent Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 26 26 27 79 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:48.716643683Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 22 22 22 T T T 1 020-039 11 11 11 26 26 26 T T T 2 040-059 8 8 8 20 20 20 T T T 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 9 9 9 21 21 21 T T T 10 200-219 6 6 6 17 17 17 T T T 11 220-239 11 11 11 26 26 26 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 10 10 10 24 24 24 F F F 14 280-299 10 10 10 24 24 24 F F F 15 300-319 5 5 5 12 12 12 T T T 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 9 9 9 21 21 21 T T T 19 380-399 10 10 10 25 25 25 F F F 20 400-419 10 10 10 25 25 25 F F F 21 420-439 10 10 10 22 22 22 T T T 22 440-459 9 9 9 21 21 21 T T T 23 460-479 10 10 10 25 25 25 F F F 24 480-499 12 12 12 27 27 27 T T T 25 500-519 10 10 10 22 22 22 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 9 9 9 21 21 21 T T T 29 580-599 10 10 10 22 22 22 T T T Max gen: 12, Max flush: 27 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 28 28 28 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 13 13 13 28 28 28 F F F 3 060-079 6 6 6 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 8 8 8 19 19 19 F F F 9 180-199 13 13 13 28 28 28 F F F 10 200-219 13 13 13 28 28 28 F F F 11 220-239 11 11 11 26 26 26 F F F 12 240-259 4 4 4 10 10 10 F F F 13 260-279 10 10 10 24 24 24 F F F 14 280-299 10 10 10 24 24 24 F F F 15 300-319 13 13 13 28 28 28 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 13 13 13 28 28 28 F F F 19 380-399 10 10 10 25 25 25 F F F 20 400-419 10 10 10 25 25 25 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 10 10 10 22 22 22 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 13 13 13 28 28 28 F F F 28 560-579 13 13 13 28 28 28 F F F 29 580-599 13 13 13 28 28 28 F F F Max gen: 13, Max flush: 28 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.032179491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.076754121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.077401721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.077439106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.07748956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.077518056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} {"msg":"Crucible 7db53e9c-f0d0-45f8-a1e8-8361fc17a91d has session id: fb3db613-89f4-4262-91ec-b1a756763420","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.077962061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Upstairs opts: Upstairs UUID: 7db53e9c-f0d0-45f8-a1e8-8361fc17a91d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078006546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Crucible stats registered with UUID: 7db53e9c-f0d0-45f8-a1e8-8361fc17a91d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078036762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:53.078062709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078593663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078625689Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} {"msg":"7db53e9c-f0d0-45f8-a1e8-8361fc17a91d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078688851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078710518Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.07877839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078824284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.07886084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078895375Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078930901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078948349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078964527Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.078980625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_connection connected from Ok(127.0.0.1:40742)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079127047Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_connection connected from Ok(127.0.0.1:65040)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079249051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_connection connected from Ok(127.0.0.1:53345)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079280837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079756668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079786405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079807322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.07982205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079836038Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.079849877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.080426895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.080499436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.080821776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081130747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081158334Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081173482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081186451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081200139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081213567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081400174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081438069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[0]R flush_numbers[0..12]: [28, 28, 28, 17, 1, 1, 21, 14, 19, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081468965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[0]R generation[0..12]: [13, 13, 13, 6, 1, 1, 9, 6, 8, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081495672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081522179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[1]R flush_numbers[0..12]: [28, 28, 28, 17, 1, 1, 21, 14, 19, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081546706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[1]R generation[0..12]: [13, 13, 13, 6, 1, 1, 9, 6, 8, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081570813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081597889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[2]R flush_numbers[0..12]: [28, 28, 28, 17, 1, 1, 21, 14, 19, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081625586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[2]R generation[0..12]: [13, 13, 13, 6, 1, 1, 9, 6, 8, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081651263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081681329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Max found gen is 14","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-12-11T23:40:53.081716825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081748171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081776177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081803464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"":"downstairs","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.0818322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081858737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"0","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081891993Z","hostname":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"1","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081926528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"client":"2","":"downstairs","session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081954645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081971953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"7db53e9c-f0d0-45f8-a1e8-8361fc17a91d is now active with session: fb3db613-89f4-4262-91ec-b1a756763420","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.081985461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.082001949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238,"session_id":"fb3db613-89f4-4262-91ec-b1a756763420"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:53.082023756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 83 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 224 len 32768 data: 2 2 2 2 2 2 2 3 03/30 Write block 395 len 40960 data: 3 3 4 4 5 3 4 4 4 4 04/30 Read block 152 len 16384 05/30 Write block 24 len 36864 data: 5 6 5 6 6 7 6 6 5 06/30 Flush 07/30 Write block 321 len 16384 data: 2 2 2 2 08/30 Read block 36 len 16384 09/30 Read block 224 len 20480 10/30 Read block 238 len 4096 11/30 Write block 211 len 32768 data: 4 4 4 4 3 3 2 2 12/30 Write block 500 len 20480 data: 4 3 4 4 4 13/30 Write block 96 len 12288 data: 2 2 2 14/30 Write block 256 len 40960 data: 3 3 3 3 3 4 4 5 5 5 15/30 Flush 16/30 Write block 71 len 32768 data: 4 2 2 2 2 2 2 2 17/30 Write block 176 len 36864 data: 3 3 3 2 3 5 5 4 4 18/30 Read block 441 len 16384 19/30 Read block 70 len 36864 20/30 Read block 530 len 8192 21/30 Write block 53 len 4096 data: 2 22/30 Write block 589 len 12288 data: 3 2 2 23/30 Write block 153 len 20480 data: 3 3 3 3 3 24/30 Write block 417 len 12288 data: 2 2 2 25/30 Flush 26/30 Write block 284 len 32768 data: 2 2 2 2 3 3 3 3 27/30 Write block 149 len 24576 data: 2 2 2 2 4 4 28/30 Read block 416 len 16384 29/30 Read block 10 len 8192 30/30 Write block 135 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:53.100669619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 28 28 28 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 14 14 14 31 31 31 F F F 3 060-079 14 14 14 31 31 31 F F F 4 080-099 14 14 14 30 30 30 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 21 21 21 T T T 7 140-159 14 14 14 31 31 31 T T T 8 160-179 14 14 14 31 31 31 F F F 9 180-199 14 14 14 31 31 31 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 14 14 14 30 30 30 F F F 13 260-279 14 14 14 30 30 30 F F F 14 280-299 10 10 10 24 24 24 T T T 15 300-319 13 13 13 28 28 28 F F F 16 320-339 14 14 14 30 30 30 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 13 13 13 28 28 28 F F F 19 380-399 14 14 14 29 29 29 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 13 13 13 28 28 28 F F F 28 560-579 13 13 13 28 28 28 F F F 29 580-599 14 14 14 31 31 31 F F F Max gen: 14, Max flush: 31 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 28 28 28 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 14 14 14 31 31 31 F F F 3 060-079 14 14 14 31 31 31 F F F 4 080-099 14 14 14 30 30 30 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 15 15 15 32 32 32 F F F 7 140-159 15 15 15 32 32 32 F F F 8 160-179 14 14 14 31 31 31 F F F 9 180-199 14 14 14 31 31 31 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 14 14 14 30 30 30 F F F 13 260-279 14 14 14 30 30 30 F F F 14 280-299 15 15 15 32 32 32 F F F 15 300-319 13 13 13 28 28 28 F F F 16 320-339 14 14 14 30 30 30 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 13 13 13 28 28 28 F F F 19 380-399 14 14 14 29 29 29 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 13 13 13 28 28 28 F F F 28 560-579 13 13 13 28 28 28 F F F 29 580-599 14 14 14 31 31 31 F F F Max gen: 15, Max flush: 32 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.326722198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.370232961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.370913836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.370951022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.3709654Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.370980378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} {"msg":"Crucible 0759fed3-4e94-48d7-aaf1-8cd953647bf9 has session id: a2044aa9-da0a-45c6-b80c-7f1ab7491e96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371375929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Upstairs opts: Upstairs UUID: 0759fed3-4e94-48d7-aaf1-8cd953647bf9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371411174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Crucible stats registered with UUID: 0759fed3-4e94-48d7-aaf1-8cd953647bf9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371427252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:57.371440441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371882686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371911822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} {"msg":"0759fed3-4e94-48d7-aaf1-8cd953647bf9 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.371989433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.37200954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372032117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372051505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372073782Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372100709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372126266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372151733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372175539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372197527Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_connection connected from Ok(127.0.0.1:61491)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372229713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_connection connected from Ok(127.0.0.1:64362)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372260009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_connection connected from Ok(127.0.0.1:42309)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372288836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372890061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.372917277Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.373025284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.373045911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.37353646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.373659955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374176821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374205577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374221225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374322433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374351869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374366577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374942946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.374969462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.37547259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.37595157Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.375973167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[0]R flush_numbers[0..12]: [28, 29, 31, 31, 30, 1, 32, 32, 31, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.375987956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[0]R generation[0..12]: [13, 14, 14, 14, 14, 1, 15, 15, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376028261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376045069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[1]R flush_numbers[0..12]: [28, 29, 31, 31, 30, 1, 32, 32, 31, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376063786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[1]R generation[0..12]: [13, 14, 14, 14, 14, 1, 15, 15, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376078114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376092353Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[2]R flush_numbers[0..12]: [28, 29, 31, 31, 30, 1, 32, 32, 31, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376106221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[2]R generation[0..12]: [13, 14, 14, 14, 14, 1, 15, 15, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376119939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376134067Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376147226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376160484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376173723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376186831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":Wait for a query_work_queue command to finish before sending IO 1246,"":"downstairs","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376206989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376219317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"0","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376232625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"1","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376245994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"client":"2","":"downstairs","session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376259402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.3762743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"0759fed3-4e94-48d7-aaf1-8cd953647bf9 is now active with session: a2044aa9-da0a-45c6-b80c-7f1ab7491e96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376287428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376299807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246,"session_id":"a2044aa9-da0a-45c6-b80c-7f1ab7491e96"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:40:57.376312325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 284 len 16384 data: 3 3 3 3 02/30 Write block 382 len 24576 data: 2 2 2 2 2 2 03/30 Read block 3 len 40960 04/30 Write block 262 len 40960 data: 5 6 6 6 4 4 3 3 2 2 05/30 Read block 243 len 28672 06/30 Read block 178 len 24576 07/30 Read block 420 len 24576 08/30 Read block 218 len 12288 09/30 Flush 10/30 Write block 60 len 16384 data: 4 4 4 2 11/30 Write block 319 len 28672 data: 2 3 3 3 3 3 2 12/30 Read block 134 len 12288 13/30 Write block 113 len 16384 data: 2 2 2 2 14/30 Read block 154 len 32768 15/30 Write block 138 len 16384 data: 2 2 2 2 16/30 Write block 556 len 40960 data: 2 3 3 3 3 4 4 4 5 3 17/30 Read block 520 len 40960 18/30 Read block 425 len 24576 19/30 Read block 376 len 4096 20/30 Write block 376 len 32768 data: 2 3 2 2 2 2 3 3 21/30 Write block 591 len 28672 data: 3 3 4 4 4 3 3 22/30 Read block 355 len 36864 23/30 Write block 174 len 4096 data: 2 24/30 Read block 138 len 36864 25/30 Write block 53 len 28672 data: 3 3 3 3 4 4 5 26/30 Flush 27/30 Write block 303 len 8192 data: 3 3 28/30 Read block 485 len 40960 29/30 Read block 523 len 40960 30/30 Read block 491 len 28672 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:40:57.397207639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 28 28 28 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 16 16 16 34 34 34 F F F 4 080-099 14 14 14 30 30 30 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 16 16 16 34 34 34 F F F 7 140-159 16 16 16 34 34 34 F F F 8 160-179 16 16 16 34 34 34 F F F 9 180-199 14 14 14 31 31 31 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 14 14 14 30 30 30 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 16 16 16 34 34 34 T T T 16 320-339 16 16 16 34 34 34 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 16 16 16 34 34 34 F F F 19 380-399 16 16 16 34 34 34 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 16 16 16 34 34 34 F F F Max gen: 16, Max flush: 34 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 28 28 28 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 16 16 16 34 34 34 F F F 4 080-099 14 14 14 30 30 30 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 16 16 16 34 34 34 F F F 7 140-159 16 16 16 34 34 34 F F F 8 160-179 16 16 16 34 34 34 F F F 9 180-199 14 14 14 31 31 31 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 14 14 14 30 30 30 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 17 17 17 35 35 35 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 16 16 16 34 34 34 F F F 19 380-399 16 16 16 34 34 34 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 F F F 23 460-479 10 10 10 25 25 25 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 16 16 16 34 34 34 F F F Max gen: 17, Max flush: 35 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.616293803Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.660526866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661183725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.66122333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661244177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661260185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} {"msg":"Crucible be6e34ef-3181-4a3e-a840-96d00132b824 has session id: f0251016-4e1c-4876-a4f5-f5dd544b7013","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661681683Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Upstairs opts: Upstairs UUID: be6e34ef-3181-4a3e-a840-96d00132b824, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661712129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Crucible stats registered with UUID: be6e34ef-3181-4a3e-a840-96d00132b824","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.661726837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:01.661740755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662275519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662301546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} {"msg":"be6e34ef-3181-4a3e-a840-96d00132b824 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662368687Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662387265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662418001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662446388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662474894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662503281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662527118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662554974Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662580841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662606008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_connection connected from Ok(127.0.0.1:45982)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662631435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_connection connected from Ok(127.0.0.1:35445)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662656392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_connection connected from Ok(127.0.0.1:40294)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.662766848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663222051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663252667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663363634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663418937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663815448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.663951281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664098682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.66411913Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664419462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664449729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664464987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664527529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664547487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664562035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.664807724Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.66532685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665354456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[0]R flush_numbers[0..12]: [28, 29, 34, 34, 30, 34, 34, 34, 34, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665370334Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[0]R generation[0..12]: [13, 14, 16, 16, 14, 16, 16, 16, 16, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665384793Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665399141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[1]R flush_numbers[0..12]: [28, 29, 34, 34, 30, 34, 34, 34, 34, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665413009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[1]R generation[0..12]: [13, 14, 16, 16, 14, 16, 16, 16, 16, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665426597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665440676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[2]R flush_numbers[0..12]: [28, 29, 34, 34, 30, 34, 34, 34, 34, 31, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665454444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[2]R generation[0..12]: [13, 14, 16, 16, 14, 16, 16, 16, 16, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665471872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.66548622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665499578Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665512957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:01.665526065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665545393Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"":"downstairs","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665558711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665571019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"0","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665584508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"1","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665597806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"client":"2","":"downstairs","session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665611294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665623683Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"be6e34ef-3181-4a3e-a840-96d00132b824 is now active with session: f0251016-4e1c-4876-a4f5-f5dd544b7013","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665636621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.66564894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254,"session_id":"f0251016-4e1c-4876-a4f5-f5dd544b7013"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:01.665661608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 242 len 28672 data: 2 2 2 2 2 2 2 03/30 Write block 8 len 36864 data: 3 3 3 3 4 4 4 3 3 04/30 Read block 411 len 16384 05/30 Read block 397 len 12288 06/30 Write block 148 len 8192 data: 2 3 07/30 Flush 08/30 Write block 85 len 8192 data: 3 3 09/30 Read block 266 len 24576 10/30 Read block 93 len 8192 11/30 Write block 176 len 40960 data: 4 4 4 3 4 6 6 5 5 4 12/30 Write block 459 len 12288 data: 2 2 2 13/30 Read block 130 len 4096 14/30 Read block 456 len 20480 15/30 Read block 343 len 24576 16/30 Read block 530 len 28672 17/30 Read block 538 len 4096 18/30 Read block 472 len 36864 19/30 Write block 394 len 36864 data: 3 4 4 5 5 6 4 5 5 20/30 Write block 240 len 16384 data: 2 2 3 3 21/30 Read block 376 len 16384 22/30 Write block 123 len 24576 data: 2 2 2 2 2 2 23/30 Write block 85 len 32768 data: 4 4 3 3 3 2 2 2 24/30 Read block 22 len 36864 25/30 Read block 235 len 4096 26/30 Write block 79 len 16384 data: 2 2 2 2 27/30 Read block 140 len 36864 28/30 Read block 81 len 8192 29/30 Read block 437 len 16384 30/30 Read block 352 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 23 23 22 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:01.684965759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 16 16 16 34 34 34 T T T 4 080-099 14 14 14 30 30 30 T T T 5 100-119 16 16 16 34 34 34 F F F 6 120-139 16 16 16 34 34 34 T T T 7 140-159 18 18 18 37 37 37 F F F 8 160-179 16 16 16 34 34 34 T T T 9 180-199 14 14 14 31 31 31 T T T 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 18 18 18 37 37 37 T T T 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 17 17 17 35 35 35 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 16 16 16 34 34 34 F F F 19 380-399 16 16 16 34 34 34 T T T 20 400-419 14 14 14 31 31 31 T T T 21 420-439 13 13 13 28 28 28 F F F 22 440-459 13 13 13 28 28 28 T T T 23 460-479 10 10 10 25 25 25 T T T 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 16 16 16 34 34 34 F F F Max gen: 18, Max flush: 37 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 F F F 1 020-039 14 14 14 29 29 29 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 19 19 19 38 38 38 F F F 4 080-099 19 19 19 38 38 38 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 19 19 19 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 19 19 19 38 38 38 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 17 17 17 35 35 35 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 6 6 6 13 13 13 F F F 18 360-379 16 16 16 34 34 34 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 13 13 13 28 28 28 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 19 19 19 38 38 38 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 16 16 16 34 34 34 F F F Max gen: 19, Max flush: 38 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.950964793Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.994608669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995268298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995308233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.99532916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995346818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} {"msg":"Crucible bfcc8b11-a70d-4aa2-a6ed-86d039766840 has session id: 42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995753617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Upstairs opts: Upstairs UUID: bfcc8b11-a70d-4aa2-a6ed-86d039766840, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995801171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Crucible stats registered with UUID: bfcc8b11-a70d-4aa2-a6ed-86d039766840","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.995828868Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:05.995849195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.99637482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996400057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} {"msg":"bfcc8b11-a70d-4aa2-a6ed-86d039766840 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996468418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996487216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996514653Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996543099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996572645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996650426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996682892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996716738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996745814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996837443Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53562)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996872698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46685)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996896935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44488)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.996912643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.997540045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.99758161Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.997603787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.997618646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.997897941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.997922718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.998316979Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.998346145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.998711859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.998956059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.998996114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.99902712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999054287Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999079224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999094262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999214637Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999235205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[0]R flush_numbers[0..12]: [37, 29, 34, 38, 38, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999249523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[0]R generation[0..12]: [18, 14, 16, 19, 19, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999263701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999278059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[1]R flush_numbers[0..12]: [37, 29, 34, 38, 38, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999292018Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[1]R generation[0..12]: [18, 14, 16, 19, 19, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999305666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999319804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[2]R flush_numbers[0..12]: [37, 29, 34, 38, 38, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999333552Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[2]R generation[0..12]: [18, 14, 16, 19, 19, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.99935089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999365218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999378597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999391905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999405013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999418172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"":"downstairs","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:05.9994313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999447888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"0","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999461277Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"1","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999474575Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"client":"2","":"downstairs","session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999488083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999500632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"bfcc8b11-a70d-4aa2-a6ed-86d039766840 is now active with session: 42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.9995135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999525749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262,"session_id":"42ae5d0a-f04a-4841-a0bb-8a8b24c45b2a"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:05.999538147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 16384 data: 3 3 3 3 02/30 Read block 160 len 40960 03/30 Read block 242 len 32768 04/30 Read block 430 len 12288 05/30 Write block 359 len 16384 data: 2 2 3 3 06/30 Write block 591 len 16384 data: 4 4 5 5 07/30 Read block 47 len 16384 08/30 Read block 7 len 4096 09/30 Write block 20 len 36864 data: 5 5 5 5 6 7 6 7 7 10/30 Read block 147 len 36864 11/30 Read block 74 len 40960 12/30 Write block 434 len 24576 data: 2 3 4 3 4 4 13/30 Write block 86 len 24576 data: 5 4 4 4 3 3 14/30 Read block 181 len 16384 15/30 Write block 14 len 40960 data: 5 4 4 3 4 5 6 6 6 6 16/30 Read block 12 len 4096 17/30 Read block 71 len 32768 18/30 Read block 476 len 4096 19/30 Write block 68 len 32768 data: 4 4 4 5 3 3 3 3 20/30 Read block 374 len 32768 21/30 Write block 72 len 28672 data: 4 4 4 4 3 3 3 22/30 Write block 419 len 28672 data: 3 3 4 4 4 3 2 23/30 Read block 329 len 12288 24/30 Write block 50 len 40960 data: 2 2 2 4 4 4 4 5 5 6 25/30 Write block 361 len 12288 data: 4 4 3 26/30 Write block 308 len 24576 data: 4 4 3 3 2 3 27/30 Read block 216 len 36864 28/30 Write block 466 len 28672 data: 2 2 2 2 2 2 3 29/30 Write block 311 len 40960 data: 4 3 4 2 2 2 2 2 3 4 30/30 Read block 439 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 28 30 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:06.020797755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 T T T 1 020-039 14 14 14 29 29 29 T T T 2 040-059 16 16 16 34 34 34 T T T 3 060-079 19 19 19 38 38 38 T T T 4 080-099 19 19 19 38 38 38 T T T 5 100-119 16 16 16 34 34 34 F F F 6 120-139 19 19 19 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 19 19 19 38 38 38 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 17 17 17 35 35 35 T T T 16 320-339 16 16 16 34 34 34 T T T 17 340-359 6 6 6 13 13 13 T T T 18 360-379 16 16 16 34 34 34 T T T 19 380-399 19 19 19 38 38 38 F F F 20 400-419 19 19 19 38 38 38 T T T 21 420-439 13 13 13 28 28 28 T T T 22 440-459 19 19 19 38 38 38 F F F 23 460-479 19 19 19 38 38 38 T T T 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 16 16 16 34 34 34 T T T Max gen: 19, Max flush: 38 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 39 39 39 F F F 1 020-039 20 20 20 39 39 39 F F F 2 040-059 20 20 20 39 39 39 F F F 3 060-079 20 20 20 39 39 39 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 19 19 19 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 F F F 12 240-259 19 19 19 38 38 38 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 20 20 20 39 39 39 F F F 16 320-339 20 20 20 39 39 39 F F F 17 340-359 20 20 20 39 39 39 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 20 20 20 39 39 39 F F F 21 420-439 20 20 20 39 39 39 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 13 13 13 28 28 28 F F F 25 500-519 14 14 14 30 30 30 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 16 16 16 34 34 34 F F F 28 560-579 16 16 16 34 34 34 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 20, Max flush: 39 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.301086223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.346295096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347209132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347250457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347265835Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347280833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} {"msg":"Crucible afd995e7-3eff-4fb0-a55e-b452946e1357 has session id: fb1dd3f2-f55d-4777-9d74-7a4c22c5880b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347702401Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Upstairs opts: Upstairs UUID: afd995e7-3eff-4fb0-a55e-b452946e1357, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347747685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Crucible stats registered with UUID: afd995e7-3eff-4fb0-a55e-b452946e1357","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.347772622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:10.347796049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348325053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.34835299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} {"msg":"afd995e7-3eff-4fb0-a55e-b452946e1357 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348413122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.3484322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348455187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348469625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348483524Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348500971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.34851499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348528378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348542246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348556075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63778)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348712395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37203)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.34875042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45227)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.348778587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349456043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349493658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349524154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349554071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349583607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.349607974Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350170584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.35020241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350224127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.35076791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350808535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350831022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.35084465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350858848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350872537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350884755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350898173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[0]R flush_numbers[0..12]: [39, 39, 39, 39, 39, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350913142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[0]R generation[0..12]: [20, 20, 20, 20, 20, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.35092713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350941368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[1]R flush_numbers[0..12]: [39, 39, 39, 39, 39, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350955296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[1]R generation[0..12]: [20, 20, 20, 20, 20, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350969185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.350987972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[2]R flush_numbers[0..12]: [39, 39, 39, 39, 39, 34, 38, 37, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351020328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[2]R generation[0..12]: [20, 20, 20, 20, 20, 16, 19, 18, 19, 19, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351036056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351050354Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351063713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:10.351077111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351095109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351108597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"":"downstairs","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351122036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351134294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"0","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351147942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"1","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351161221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"client":"2","":"downstairs","session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351174699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351187117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"afd995e7-3eff-4fb0-a55e-b452946e1357 is now active with session: fb1dd3f2-f55d-4777-9d74-7a4c22c5880b","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-11T23:41:10.351200136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351217944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270,"session_id":"fb1dd3f2-f55d-4777-9d74-7a4c22c5880b"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:10.351231042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 61 len 20480 data: 5 5 3 2 2 02/30 Write block 344 len 8192 data: 2 2 03/30 Write block 48 len 8192 data: 2 2 04/30 Read block 102 len 20480 05/30 Read block 504 len 12288 06/30 Read block 238 len 12288 07/30 Write block 514 len 40960 data: 3 4 4 5 4 4 3 3 2 2 08/30 Write block 553 len 32768 data: 3 3 2 3 4 4 4 4 09/30 Flush 10/30 Write block 223 len 40960 data: 2 3 3 3 3 3 3 3 4 4 11/30 Read block 142 len 4096 12/30 Write block 560 len 12288 data: 5 5 5 13/30 Read block 402 len 36864 14/30 Write block 14 len 4096 data: 6 15/30 Write block 486 len 20480 data: 3 3 3 3 3 16/30 Write block 40 len 36864 data: 3 2 2 2 2 2 2 2 3 17/30 Read block 385 len 40960 18/30 Read block 231 len 16384 19/30 Read block 216 len 36864 20/30 Read block 567 len 28672 21/30 Read block 418 len 12288 22/30 Read block 54 len 32768 23/30 Read block 78 len 28672 24/30 Write block 425 len 24576 data: 3 2 2 2 2 2 25/30 Write block 326 len 12288 data: 2 2 2 26/30 Write block 542 len 12288 data: 2 2 2 27/30 Write block 241 len 16384 data: 3 4 4 3 28/30 Read block 257 len 16384 29/30 Write block 223 len 16384 data: 3 4 4 4 30/30 Read block 524 len 16384 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 21 20 19 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:10.370787232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 39 39 39 T T T 1 020-039 20 20 20 39 39 39 F F F 2 040-059 22 22 22 40 40 40 T T T 3 060-079 22 22 22 40 40 40 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 19 19 19 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 14 14 14 29 29 29 T T T 12 240-259 19 19 19 38 38 38 T T T 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 20 20 20 39 39 39 F F F 16 320-339 20 20 20 39 39 39 T T T 17 340-359 22 22 22 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 20 20 20 39 39 39 F F F 21 420-439 20 20 20 39 39 39 T T T 22 440-459 19 19 19 38 38 38 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 13 13 13 28 28 28 T T T 25 500-519 22 22 22 40 40 40 F F F 26 520-539 22 22 22 40 40 40 F F F 27 540-559 22 22 22 40 40 40 T T T 28 560-579 22 22 22 40 40 40 T T T 29 580-599 20 20 20 39 39 39 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 41 41 41 F F F 1 020-039 20 20 20 39 39 39 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 19 19 19 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 14 14 14 30 30 30 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 F F F 15 300-319 20 20 20 39 39 39 F F F 16 320-339 23 23 23 41 41 41 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 20 20 20 39 39 39 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 23 23 23 41 41 41 F F F 25 500-519 22 22 22 40 40 40 F F F 26 520-539 22 22 22 40 40 40 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.63032187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67361139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674194498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674236203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.6742545Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674270718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} {"msg":"Crucible 62c2b27b-6fd4-4cba-86c4-0c890124c086 has session id: abf98ef7-63f9-4cfc-9d36-77f62df7be21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674711403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Upstairs opts: Upstairs UUID: 62c2b27b-6fd4-4cba-86c4-0c890124c086, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674744809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Crucible stats registered with UUID: 62c2b27b-6fd4-4cba-86c4-0c890124c086","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.674760307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:14.674773356Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675366192Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675391239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} {"msg":"62c2b27b-6fd4-4cba-86c4-0c890124c086 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675455441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675473609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675496366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675520453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67554732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675574146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675606732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675649667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675680143Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675710089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_connection connected from Ok(127.0.0.1:38458)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675738656Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_connection connected from Ok(127.0.0.1:46488)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.675763203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_connection connected from Ok(127.0.0.1:48994)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67578485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.676337001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.676367927Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.676391864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.676420451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67699004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677034575Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677343246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677384111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677548251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677586176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677615112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677641259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677669986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677690673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.677937232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678404274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678432671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[0]R flush_numbers[0..12]: [41, 39, 41, 40, 39, 34, 38, 37, 38, 38, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678448779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[0]R generation[0..12]: [23, 20, 23, 22, 20, 16, 19, 18, 19, 19, 14, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678463367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678478115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[1]R flush_numbers[0..12]: [41, 39, 41, 40, 39, 34, 38, 37, 38, 38, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678492503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[1]R generation[0..12]: [23, 20, 23, 22, 20, 16, 19, 18, 19, 19, 14, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678506652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67852091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[2]R flush_numbers[0..12]: [41, 39, 41, 40, 39, 34, 38, 37, 38, 38, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678538658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[2]R generation[0..12]: [23, 20, 23, 22, 20, 16, 19, 18, 19, 19, 14, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678552916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678567244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678580703Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678594081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678607369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678620707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"":"downstairs","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678633986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Transition from WaitQuorum to Active","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678655243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"0","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678669032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"1","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.67868226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"client":"2","":"downstairs","session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678695598Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678708027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"62c2b27b-6fd4-4cba-86c4-0c890124c086 is now active with session: abf98ef7-63f9-4cfc-9d36-77f62df7be21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678720995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678733373Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278,"session_id":"abf98ef7-63f9-4cfc-9d36-77f62df7be21"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:14.678748232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 8192 data: 3 3 02/30 Write block 465 len 32768 data: 2 3 3 3 3 3 3 4 03/30 Write block 504 len 28672 data: 5 4 4 3 3 2 2 04/30 Write block 145 len 32768 data: 2 2 2 3 4 3 3 3 05/30 Write block 31 len 24576 data: 7 6 5 5 4 4 06/30 Write block 444 len 16384 data: 5 4 4 4 07/30 Read block 516 len 36864 08/30 Read block 291 len 8192 09/30 Write block 132 len 4096 data: 3 10/30 Read block 567 len 4096 11/30 Read block 501 len 12288 12/30 Write block 147 len 12288 data: 3 4 5 13/30 Write block 310 len 20480 data: 4 5 4 5 3 14/30 Write block 152 len 20480 data: 4 5 5 4 4 15/30 Write block 207 len 28672 data: 2 2 2 3 5 5 5 16/30 Write block 404 len 24576 data: 5 3 3 3 2 2 17/30 Read block 395 len 8192 18/30 Write block 434 len 8192 data: 3 4 19/30 Flush 20/30 Write block 127 len 36864 data: 3 3 2 3 3 4 3 3 3 21/30 Write block 312 len 8192 data: 5 6 22/30 Read block 47 len 28672 23/30 Write block 535 len 32768 data: 2 2 2 2 2 2 2 3 24/30 Read block 172 len 4096 25/30 Read block 195 len 28672 26/30 Read block 372 len 40960 27/30 Read block 485 len 4096 28/30 Write block 281 len 32768 data: 2 2 2 4 4 4 4 4 29/30 Write block 11 len 20480 data: 4 5 5 7 5 30/30 Write block 477 len 40960 data: 3 3 3 3 4 4 5 3 3 4 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:14.696936971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 41 41 41 T T T 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 24 24 24 42 42 42 T T T 7 140-159 24 24 24 42 42 42 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 24 24 24 42 42 42 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 16 16 16 33 33 33 T T T 15 300-319 24 24 24 42 42 42 T T T 16 320-339 23 23 23 41 41 41 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 24 24 24 42 42 42 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 24 24 24 42 42 42 T T T 24 480-499 23 23 23 41 41 41 T T T 25 500-519 24 24 24 42 42 42 F F F 26 520-539 22 22 22 40 40 40 T T T 27 540-559 23 23 23 41 41 41 T T T 28 560-579 23 23 23 41 41 41 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 24, Max flush: 42 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 43 43 43 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 25 25 25 43 43 43 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 24 24 24 42 42 42 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 25 25 25 43 43 43 F F F 15 300-319 25 25 25 43 43 43 F F F 16 320-339 23 23 23 41 41 41 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 24 24 24 42 42 42 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 25 25 25 43 43 43 F F F 24 480-499 25 25 25 43 43 43 F F F 25 500-519 24 24 24 42 42 42 F F F 26 520-539 25 25 25 43 43 43 F F F 27 540-559 25 25 25 43 43 43 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 20 20 20 39 39 39 F F F Max gen: 25, Max flush: 43 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.955471122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.998796488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.999493442Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.999530507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.999549624Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:18.999564782Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} {"msg":"Crucible f78882c7-1469-4817-a32d-640e842798d7 has session id: 8764c04e-095c-4321-a5db-762672df955c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000035674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Upstairs opts: Upstairs UUID: f78882c7-1469-4817-a32d-640e842798d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000076809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Crucible stats registered with UUID: f78882c7-1469-4817-a32d-640e842798d7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000104086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:19.000128923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000637409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000663026Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} {"msg":"f78882c7-1469-4817-a32d-640e842798d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000726058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000745156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000768853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.00079487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000821906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000851323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000879859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000908966Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000943491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000972368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44212)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.000998395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43442)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001024171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53475)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001050638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001695078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001722185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001737712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.001751751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002264297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002293043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002337308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002361505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002866792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002896139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002916776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.002991967Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003032482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003060818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003083175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.00352794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003564685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[0]R flush_numbers[0..12]: [43, 42, 41, 40, 39, 34, 43, 42, 38, 38, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003591222Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[0]R generation[0..12]: [25, 24, 23, 22, 20, 16, 25, 24, 19, 19, 24, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003616599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003643046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[1]R flush_numbers[0..12]: [43, 42, 41, 40, 39, 34, 43, 42, 38, 38, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003667523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[1]R generation[0..12]: [25, 24, 23, 22, 20, 16, 25, 24, 19, 19, 24, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.00369171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003717737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[2]R flush_numbers[0..12]: [43, 42, 41, 40, 39, 34, 43, 42, 38, 38, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003745063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[2]R generation[0..12]: [25, 24, 23, 22, 20, 16, 25, 24, 19, 19, 24, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003775149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003805426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Max found gen is 26"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003839771Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003869068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003896174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003923221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"":"downstairs","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003950828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.003976574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"0","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004004971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"1","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-11T23:41:19.004033367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"client":"2","":"downstairs","session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004059774Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004074812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"f78882c7-1469-4817-a32d-640e842798d7 is now active with session: 8764c04e-095c-4321-a5db-762672df955c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004088171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004100639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286,"session_id":"8764c04e-095c-4321-a5db-762672df955c"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:19.004113217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 539 len 28672 data: 3 3 3 4 3 3 2 02/30 Read block 421 len 20480 03/30 Read block 130 len 28672 04/30 Write block 589 len 24576 data: 4 3 5 5 6 6 05/30 Read block 209 len 24576 06/30 Read block 292 len 8192 07/30 Read block 180 len 20480 08/30 Read block 154 len 28672 09/30 Read block 248 len 4096 10/30 Read block 118 len 24576 11/30 Read block 521 len 20480 12/30 Read block 122 len 36864 13/30 Read block 105 len 36864 14/30 Write block 192 len 32768 data: 3 5 5 4 4 4 4 3 15/30 Write block 495 len 4096 data: 3 16/30 Read block 535 len 24576 17/30 Read block 280 len 40960 18/30 Read block 519 len 4096 19/30 Read block 48 len 28672 20/30 Read block 407 len 32768 21/30 Read block 494 len 32768 22/30 Write block 326 len 20480 data: 3 3 3 2 2 23/30 Read block 383 len 16384 24/30 Write block 61 len 16384 data: 6 6 4 3 25/30 Read block 479 len 40960 26/30 Read block 234 len 36864 27/30 Write block 285 len 4096 data: 5 28/30 Write block 564 len 40960 data: 6 4 3 2 4 4 4 4 4 4 29/30 Write block 290 len 8192 data: 4 4 30/30 Write block 221 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:19.023132734Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 43 43 43 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 22 22 22 40 40 40 T T T 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 25 25 25 43 43 43 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 T T T 10 200-219 24 24 24 42 42 42 F F F 11 220-239 23 23 23 41 41 41 T T T 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 25 25 25 43 43 43 T T T 15 300-319 25 25 25 43 43 43 F F F 16 320-339 23 23 23 41 41 41 T T T 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 24 24 24 42 42 42 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 25 25 25 43 43 43 F F F 24 480-499 25 25 25 43 43 43 T T T 25 500-519 24 24 24 42 42 42 F F F 26 520-539 25 25 25 43 43 43 T T T 27 540-559 25 25 25 43 43 43 T T T 28 560-579 23 23 23 41 41 41 T T T 29 580-599 20 20 20 39 39 39 T T T Max gen: 25, Max flush: 43 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 43 43 43 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 26 26 26 44 44 44 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 16 16 16 34 34 34 F F F 6 120-139 25 25 25 43 43 43 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 26 26 26 44 44 44 F F F 10 200-219 24 24 24 42 42 42 F F F 11 220-239 26 26 26 44 44 44 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 25 25 25 43 43 43 F F F 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 24 24 24 42 42 42 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 25 25 25 43 43 43 F F F 24 480-499 26 26 26 44 44 44 F F F 25 500-519 24 24 24 42 42 42 F F F 26 520-539 26 26 26 44 44 44 F F F 27 540-559 26 26 26 44 44 44 F F F 28 560-579 26 26 26 44 44 44 F F F 29 580-599 26 26 26 44 44 44 F F F Max gen: 26, Max flush: 44 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.2862968Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:23.328188295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.328567877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.359433292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360019039Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360058914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360074102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.36008852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} {"msg":"Crucible e4e91cdb-ca15-4256-991f-79c6bc30efa7 has session id: f81c3069-8e67-4495-8919-75fe615758dd","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360552643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Upstairs opts: Upstairs UUID: e4e91cdb-ca15-4256-991f-79c6bc30efa7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360585438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Crucible stats registered with UUID: e4e91cdb-ca15-4256-991f-79c6bc30efa7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.360601927Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:23.360615115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.36105595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361082307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} {"msg":"e4e91cdb-ca15-4256-991f-79c6bc30efa7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361123182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.36114052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361162587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361177035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361190923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.36122052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361247736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361263544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361278263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361356213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52335)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361390638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:34384)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361520942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38518)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361562387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.361987564Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362016291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362168702Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362190159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362336841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362376856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362691247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.362774437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363095897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363299401Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363324078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363339186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363398559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363418317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363432135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363590875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363611282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[0]R flush_numbers[0..12]: [43, 42, 41, 44, 39, 34, 43, 42, 38, 44, 42, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363625541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[0]R generation[0..12]: [25, 24, 23, 26, 20, 16, 25, 24, 19, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363639519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363654567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[1]R flush_numbers[0..12]: [43, 42, 41, 44, 39, 34, 43, 42, 38, 44, 42, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363668406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[1]R generation[0..12]: [25, 24, 23, 26, 20, 16, 25, 24, 19, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363683084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363701521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[2]R flush_numbers[0..12]: [43, 42, 41, 44, 39, 34, 43, 42, 38, 44, 42, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.36371646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[2]R generation[0..12]: [25, 24, 23, 26, 20, 16, 25, 24, 19, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363730868Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363744936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363758144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Generation requested: 28 >= found:27","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363771343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363785341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363798549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"":"downstairs","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363812877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:23.363825156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"0","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363849033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"1","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363864621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"client":"2","":"downstairs","session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363878139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363891228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"e4e91cdb-ca15-4256-991f-79c6bc30efa7 is now active with session: f81c3069-8e67-4495-8919-75fe615758dd","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363904196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363919814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294,"session_id":"f81c3069-8e67-4495-8919-75fe615758dd"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:23.363945491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 32768 data: 2 2 2 2 2 3 3 3 02/30 Read block 16 len 12288 03/30 Read block 64 len 36864 04/30 Read block 130 len 36864 05/30 Flush 06/30 Write block 413 len 36864 data: 2 2 2 2 3 3 4 4 5 07/30 Write block 492 len 40960 data: 2 2 2 4 3 3 3 4 5 4 08/30 Read block 137 len 4096 09/30 Write block 225 len 40960 data: 5 5 4 4 4 4 5 5 3 4 10/30 Read block 492 len 8192 11/30 Write block 22 len 24576 data: 7 7 7 8 7 8 12/30 Read block 302 len 24576 13/30 Write block 203 len 4096 data: 4 14/30 Read block 220 len 8192 15/30 Flush 16/30 Flush 17/30 Write block 174 len 32768 data: 3 2 5 5 5 4 5 7 18/30 Write block 44 len 20480 data: 3 3 3 3 4 19/30 Write block 478 len 32768 data: 4 4 4 5 5 6 4 4 20/30 Write block 114 len 32768 data: 3 3 3 2 2 2 2 2 21/30 Read block 559 len 16384 22/30 Read block 236 len 4096 23/30 Read block 492 len 8192 24/30 Write block 570 len 32768 data: 5 5 5 5 4 3 3 3 25/30 Write block 307 len 4096 data: 4 26/30 Read block 8 len 16384 27/30 Write block 564 len 20480 data: 7 5 4 3 5 28/30 Write block 88 len 24576 data: 5 5 4 4 3 2 29/30 Read block 339 len 8192 30/30 Read block 466 len 8192 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:23.383302175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 28 28 28 46 46 46 F F F 2 040-059 23 23 23 41 41 41 T T T 3 060-079 26 26 26 44 44 44 F F F 4 080-099 20 20 20 39 39 39 T T T 5 100-119 16 16 16 34 34 34 T T T 6 120-139 25 25 25 43 43 43 T T T 7 140-159 24 24 24 42 42 42 F F F 8 160-179 19 19 19 38 38 38 T T T 9 180-199 26 26 26 44 44 44 T T T 10 200-219 28 28 28 46 46 46 F F F 11 220-239 28 28 28 46 46 46 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 25 25 25 43 43 43 T T T 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 28 28 28 46 46 46 F F F 21 420-439 28 28 28 46 46 46 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 25 25 25 43 43 43 T T T 24 480-499 28 28 28 46 46 46 T T T 25 500-519 28 28 28 46 46 46 F F F 26 520-539 26 26 26 44 44 44 F F F 27 540-559 26 26 26 44 44 44 F F F 28 560-579 26 26 26 44 44 44 T T T 29 580-599 26 26 26 44 44 44 F F F Max gen: 28, Max flush: 46 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 28 28 28 46 46 46 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 44 44 44 F F F 4 080-099 29 29 29 47 47 47 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 29 29 29 47 47 47 F F F 7 140-159 24 24 24 42 42 42 F F F 8 160-179 29 29 29 47 47 47 F F F 9 180-199 29 29 29 47 47 47 F F F 10 200-219 28 28 28 46 46 46 F F F 11 220-239 28 28 28 46 46 46 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 29 29 29 47 47 47 F F F 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 28 28 28 46 46 46 F F F 21 420-439 28 28 28 46 46 46 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 29 29 29 47 47 47 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 28 28 28 46 46 46 F F F 26 520-539 26 26 26 44 44 44 F F F 27 540-559 26 26 26 44 44 44 F F F 28 560-579 29 29 29 47 47 47 F F F 29 580-599 26 26 26 44 44 44 F F F Max gen: 29, Max flush: 47 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.681768975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.726211002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727045809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727099242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.72712066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727136187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} {"msg":"Crucible d81b8d1c-ccbc-4009-a2f1-b9703815d578 has session id: 372a3ca2-4b3f-4df8-9fb4-65970c61e263","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727672991Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Upstairs opts: Upstairs UUID: d81b8d1c-ccbc-4009-a2f1-b9703815d578, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727702327Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Crucible stats registered with UUID: d81b8d1c-ccbc-4009-a2f1-b9703815d578","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.727717365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:27.727730324Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728233511Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728258208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} {"msg":"d81b8d1c-ccbc-4009-a2f1-b9703815d578 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728327439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728346227Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728371074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728385672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.72839923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728412529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728431197Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728460873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728489349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728513796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_connection connected from Ok(127.0.0.1:64168)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728539563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_connection connected from Ok(127.0.0.1:36081)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.7285643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:27.728743998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:27.728787852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:27.728820068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.728935924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.729424653Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.729452959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.729598321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.729618859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730044826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730185848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730585899Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730615785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730631583Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730705804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.730725621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:27.73073987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.729965016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_connection connected from Ok(127.0.0.1:33781)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.73010177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.731257696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.7313057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.731830744Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732382846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732408043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[0]R flush_numbers[0..12]: [45, 46, 47, 44, 47, 47, 47, 42, 47, 47, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732423971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[0]R generation[0..12]: [28, 28, 29, 26, 29, 29, 29, 24, 29, 29, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732444358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732459406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[1]R flush_numbers[0..12]: [45, 46, 47, 44, 47, 47, 47, 42, 47, 47, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732473585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[1]R generation[0..12]: [28, 28, 29, 26, 29, 29, 29, 24, 29, 29, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732487483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732501941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[2]R flush_numbers[0..12]: [45, 46, 47, 44, 47, 47, 47, 42, 47, 47, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.73251591Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[2]R generation[0..12]: [28, 28, 29, 26, 29, 29, 29, 24, 29, 29, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732529908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732544216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732557674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732571383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732584941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732598219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"":"downstairs","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732612118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732624536Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"0","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732638494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"1","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732655052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"client":"2","":"downstairs","session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:37.73266939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732685968Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"d81b8d1c-ccbc-4009-a2f1-b9703815d578 is now active with session: 372a3ca2-4b3f-4df8-9fb4-65970c61e263","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732699267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732712015Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302,"session_id":"372a3ca2-4b3f-4df8-9fb4-65970c61e263"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:37.732724633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 20480 data: 2 2 2 2 3 02/30 Read block 537 len 16384 03/30 Write block 468 len 8192 data: 4 4 04/30 Read block 225 len 12288 05/30 Write block 207 len 4096 data: 3 06/30 Read block 361 len 32768 07/30 Read block 371 len 4096 08/30 Read block 227 len 20480 09/30 Read block 324 len 36864 10/30 Write block 121 len 40960 data: 3 3 3 3 3 3 4 4 3 4 11/30 Write block 590 len 8192 data: 4 6 12/30 Read block 48 len 20480 13/30 Read block 183 len 24576 14/30 Write block 362 len 32768 data: 5 4 3 2 2 2 2 3 15/30 Flush 16/30 Write block 186 len 40960 data: 4 4 4 4 3 3 4 6 6 5 17/30 Read block 291 len 8192 18/30 Write block 408 len 16384 data: 3 3 2 2 19/30 Write block 549 len 4096 data: 3 20/30 Write block 164 len 32768 data: 3 3 3 3 3 2 2 2 21/30 Write block 145 len 36864 data: 3 3 4 5 6 4 4 5 6 22/30 Read block 299 len 12288 23/30 Write block 523 len 8192 data: 3 2 24/30 Write block 211 len 4096 data: 6 25/30 Read block 477 len 16384 26/30 Read block 276 len 24576 27/30 Read block 471 len 12288 28/30 Read block 186 len 28672 29/30 Read block 69 len 8192 30/30 Write block 19 len 28672 data: 6 7 7 8 8 8 9 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:37.752561608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 T T T 1 020-039 28 28 28 46 46 46 T T T 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 44 44 44 F F F 4 080-099 29 29 29 47 47 47 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 24 24 24 42 42 42 T T T 8 160-179 29 29 29 47 47 47 T T T 9 180-199 29 29 29 47 47 47 T T T 10 200-219 30 30 30 48 48 48 T T T 11 220-239 28 28 28 46 46 46 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 29 29 29 47 47 47 F F F 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 28 28 28 46 46 46 T T T 21 420-439 28 28 28 46 46 46 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 30 30 30 48 48 48 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 28 28 28 46 46 46 F F F 26 520-539 26 26 26 44 44 44 T T T 27 540-559 30 30 30 48 48 48 T T T 28 560-579 29 29 29 47 47 47 F F F 29 580-599 30 30 30 48 48 48 F F F Max gen: 30, Max flush: 48 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 49 49 49 F F F 1 020-039 31 31 31 49 49 49 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 44 44 44 F F F 4 080-099 29 29 29 47 47 47 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 31 31 31 49 49 49 F F F 8 160-179 31 31 31 49 49 49 F F F 9 180-199 31 31 31 49 49 49 F F F 10 200-219 31 31 31 49 49 49 F F F 11 220-239 28 28 28 46 46 46 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 29 29 29 47 47 47 F F F 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 31 31 31 49 49 49 F F F 21 420-439 28 28 28 46 46 46 F F F 22 440-459 24 24 24 42 42 42 F F F 23 460-479 30 30 30 48 48 48 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 28 28 28 46 46 46 F F F 26 520-539 31 31 31 49 49 49 F F F 27 540-559 31 31 31 49 49 49 F F F 28 560-579 29 29 29 47 47 47 F F F 29 580-599 30 30 30 48 48 48 F F F Max gen: 31, Max flush: 49 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.015246214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.058520686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059282822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059322506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059338255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059354053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} {"msg":"Crucible 5fef7481-0626-4438-865b-1553e077a8f7 has session id: 93f2ba65-f0a4-4f94-b623-268a64b94de7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059767181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Upstairs opts: Upstairs UUID: 5fef7481-0626-4438-865b-1553e077a8f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059808066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Crucible stats registered with UUID: 5fef7481-0626-4438-865b-1553e077a8f7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.059832693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:42.059847841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06033844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060366947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} {"msg":"5fef7481-0626-4438-865b-1553e077a8f7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060431649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060451886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060478643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06050436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060532206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060560083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060589289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060617696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060640463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06066168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62012)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060688837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34377)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.060717813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61257)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06074557Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.061403408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.061432194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.061449672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.061475789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062084283Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062130837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062733123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062761569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062777367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062791126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062804614Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.062817812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.063267406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.063287374Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06380026Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064298548Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064323615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[0]R flush_numbers[0..12]: [49, 49, 47, 44, 47, 47, 48, 49, 49, 49, 49, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064338273Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[0]R generation[0..12]: [31, 31, 29, 26, 29, 29, 30, 31, 31, 31, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064352401Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06436679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[1]R flush_numbers[0..12]: [49, 49, 47, 44, 47, 47, 48, 49, 49, 49, 49, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064385167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[1]R generation[0..12]: [31, 31, 29, 26, 29, 29, 30, 31, 31, 31, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064399295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064413274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[2]R flush_numbers[0..12]: [49, 49, 47, 44, 47, 47, 48, 49, 49, 49, 49, 46]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064427042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[2]R generation[0..12]: [31, 31, 29, 26, 29, 29, 30, 31, 31, 31, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06444184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064456699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064470017Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064483525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064496463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064509422Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"":"downstairs","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.06452304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:42.064535159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"0","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064556746Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"1","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064570334Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"client":"2","":"downstairs","session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064584503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064600231Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"5fef7481-0626-4438-865b-1553e077a8f7 is now active with session: 93f2ba65-f0a4-4f94-b623-268a64b94de7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064613449Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064626457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310,"session_id":"93f2ba65-f0a4-4f94-b623-268a64b94de7"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:42.064638966Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 16384 data: 7 5 5 6 02/30 Write block 577 len 24576 data: 4 3 3 2 2 2 03/30 Read block 186 len 36864 04/30 Read block 301 len 28672 05/30 Read block 463 len 8192 06/30 Write block 43 len 4096 data: 3 07/30 Read block 438 len 20480 08/30 Read block 583 len 8192 09/30 Write block 559 len 20480 data: 5 6 6 6 5 10/30 Write block 70 len 12288 data: 5 6 5 11/30 Write block 164 len 12288 data: 4 4 4 12/30 Flush 13/30 Write block 229 len 28672 data: 5 5 6 6 4 5 3 14/30 Read block 317 len 20480 15/30 Read block 299 len 4096 16/30 Write block 78 len 36864 data: 4 3 3 3 3 3 3 5 6 17/30 Write block 506 len 40960 data: 5 4 4 3 3 2 2 3 4 5 18/30 Write block 7 len 8192 data: 4 4 19/30 Read block 561 len 24576 20/30 Flush 21/30 Write block 178 len 8192 data: 6 5 22/30 Read block 552 len 4096 23/30 Write block 18 len 24576 data: 5 7 8 8 9 9 24/30 Write block 183 len 12288 data: 6 6 5 25/30 Write block 313 len 20480 data: 7 4 3 3 3 26/30 Read block 393 len 12288 27/30 Read block 462 len 40960 28/30 Write block 438 len 20480 data: 5 5 5 4 5 29/30 Write block 351 len 24576 data: 4 3 3 2 2 2 30/30 Write block 164 len 32768 data: 5 5 5 4 4 3 3 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:42.082432345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 51 51 51 T T T 1 020-039 31 31 31 49 49 49 T T T 2 040-059 32 32 32 50 50 50 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 32 32 32 50 50 50 F F F 8 160-179 32 32 32 50 50 50 T T T 9 180-199 31 31 31 49 49 49 T T T 10 200-219 31 31 31 49 49 49 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 29 29 29 47 47 47 T T T 16 320-339 26 26 26 44 44 44 F F F 17 340-359 22 22 22 40 40 40 T T T 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 31 31 31 49 49 49 F F F 21 420-439 28 28 28 46 46 46 T T T 22 440-459 24 24 24 42 42 42 T T T 23 460-479 30 30 30 48 48 48 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 32 32 32 51 51 51 F F F 26 520-539 31 31 31 49 49 49 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 32 32 32 50 50 50 F F F Max gen: 32, Max flush: 51 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 52 52 52 F F F 1 020-039 33 33 33 52 52 52 F F F 2 040-059 32 32 32 50 50 50 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 32 32 32 50 50 50 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 33 33 33 52 52 52 F F F 10 200-219 31 31 31 49 49 49 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 33 33 33 52 52 52 F F F 16 320-339 26 26 26 44 44 44 F F F 17 340-359 33 33 33 52 52 52 F F F 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 31 31 31 49 49 49 F F F 21 420-439 33 33 33 52 52 52 F F F 22 440-459 33 33 33 52 52 52 F F F 23 460-479 30 30 30 48 48 48 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 32 32 32 51 51 51 F F F 26 520-539 31 31 31 49 49 49 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 32 32 32 50 50 50 F F F Max gen: 33, Max flush: 52 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.336247433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.37940735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380023803Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380066188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380081956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380097224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} {"msg":"Crucible 4a8fea7a-1bda-4502-b4d1-ea35b2bc5200 has session id: 03c1c427-84bb-4fc4-b93c-95daaeada406","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380605101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Upstairs opts: Upstairs UUID: 4a8fea7a-1bda-4502-b4d1-ea35b2bc5200, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.380647936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Crucible stats registered with UUID: 4a8fea7a-1bda-4502-b4d1-ea35b2bc5200","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38069126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:46.380715127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38117406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381212205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} {"msg":"4a8fea7a-1bda-4502-b4d1-ea35b2bc5200 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381273648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381299145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381323922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38133977Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381354128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381376775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381406761Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381432758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381459535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381486951Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_connection connected from Ok(127.0.0.1:41813)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381512448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_connection connected from Ok(127.0.0.1:55170)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381537185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_connection connected from Ok(127.0.0.1:47827)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.381675168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.382275383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38230512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.382321358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.382336096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.382931942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.382972647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.383549915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38359328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.383619546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.383641204Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.383666331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.383691647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.384183036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.384211613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.384706071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385165924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385185632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[0]R flush_numbers[0..12]: [52, 52, 50, 51, 51, 47, 48, 50, 52, 52, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38520156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[0]R generation[0..12]: [33, 33, 32, 32, 32, 29, 30, 32, 33, 33, 31, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385215798Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385230266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[1]R flush_numbers[0..12]: [52, 52, 50, 51, 51, 47, 48, 50, 52, 52, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385245124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[1]R generation[0..12]: [33, 33, 32, 32, 32, 29, 30, 32, 33, 33, 31, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385258943Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38527769Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[2]R flush_numbers[0..12]: [52, 52, 50, 51, 51, 47, 48, 50, 52, 52, 49, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385291909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[2]R generation[0..12]: [33, 33, 32, 32, 32, 29, 30, 32, 33, 33, 31, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385305717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385319745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385333043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385346412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.38535946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385372479Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"":"downstairs","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385385597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385397765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV",Wait for a query_work_queue command to finish before sending IO "pid":1318,"client":"0","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385417423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"1","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385430661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"client":"2","":"downstairs","session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385456498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"4a8fea7a-1bda-4502-b4d1-ea35b2bc5200 is now active with session: 03c1c427-84bb-4fc4-b93c-95daaeada406","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385469467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385483915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318,"session_id":"03c1c427-84bb-4fc4-b93c-95daaeada406"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:46.385496303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 24576 data: 5 3 2 2 2 4 02/30 Write block 6 len 20480 data: 4 5 5 4 4 03/30 Flush 04/30 Write block 432 len 28672 data: 2 2 4 5 5 4 6 05/30 Flush 06/30 Write block 183 len 24576 data: 7 7 6 5 5 5 07/30 Read block 390 len 20480 08/30 Read block 318 len 36864 09/30 Read block 194 len 8192 10/30 Write block 238 len 20480 data: 2 2 3 4 5 11/30 Read block 355 len 12288 12/30 Write block 205 len 16384 data: 4 3 4 3 13/30 Write block 315 len 36864 data: 4 4 4 3 4 5 4 4 4 14/30 Read block 57 len 24576 15/30 Read block 231 len 28672 16/30 Read block 564 len 32768 17/30 Write block 317 len 24576 data: 5 4 5 6 5 5 18/30 Read block 58 len 4096 19/30 Read block 547 len 8192 20/30 Read block 479 len 28672 21/30 Read block 60 len 24576 22/30 Read block 67 len 28672 23/30 Read block 429 len 36864 24/30 Read block 144 len 16384 25/30 Read block 473 len 16384 26/30 Write block 403 len 32768 data: 5 6 4 4 4 4 4 3 27/30 Write block 502 len 8192 data: 5 5 28/30 Read block 271 len 32768 29/30 Read block 294 len 32768 30/30 Read block 235 len 40960 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 6 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 25 25 24 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:46.406079325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 53 53 53 F F F 1 020-039 33 33 33 52 52 52 F F F 2 040-059 32 32 32 50 50 50 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 32 32 32 50 50 50 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 33 33 33 52 52 52 T T T 10 200-219 31 31 31 49 49 49 T T T 11 220-239 32 32 32 51 51 51 T T T 12 240-259 23 23 23 41 41 41 T T T 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 33 33 33 52 52 52 T T T 16 320-339 26 26 26 44 44 44 T T T 17 340-359 33 33 33 52 52 52 F F F 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 31 31 31 49 49 49 T T T 21 420-439 34 34 34 54 54 54 F F F 22 440-459 33 33 33 52 52 52 F F F 23 460-479 34 34 34 53 53 53 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 32 32 32 51 51 51 T T T 26 520-539 31 31 31 49 49 49 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 32 32 32 50 50 50 F F F Max gen: 34, Max flush: 54 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 53 53 53 F F F 1 020-039 33 33 33 52 52 52 F F F 2 040-059 32 32 32 50 50 50 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 32 32 32 50 50 50 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 35 35 35 55 55 55 F F F 10 200-219 35 35 35 55 55 55 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 35 35 35 55 55 55 F F F 16 320-339 35 35 35 55 55 55 F F F 17 340-359 33 33 33 52 52 52 F F F 18 360-379 30 30 30 48 48 48 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 35 35 35 55 55 55 F F F 21 420-439 34 34 34 54 54 54 F F F 22 440-459 33 33 33 52 52 52 F F F 23 460-479 34 34 34 53 53 53 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 35 35 35 55 55 55 F F F 26 520-539 31 31 31 49 49 49 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 32 32 32 50 50 50 F F F Max gen: 35, Max flush: 55 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.662547344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.707606715Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.708551987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.708596852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.70861218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.708628358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} {"msg":"Crucible 0b0629ee-4844-4480-a573-93b7615247bb has session id: 4a3678ca-9dd6-4102-a2d8-7caade64f5c4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709211235Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Upstairs opts: Upstairs UUID: 0b0629ee-4844-4480-a573-93b7615247bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.70925131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Crucible stats registered with UUID: 0b0629ee-4844-4480-a573-93b7615247bb","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709270638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:50.709285386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709848196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709872603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} {"msg":"0b0629ee-4844-4480-a573-93b7615247bb active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709932196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.709949753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.70997945Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710006566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710030103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.71005858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710085356Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710100675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710115033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710129061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:57768)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710143229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42955)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710157148Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43537)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710250656Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.710991824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711033229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711055196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711074804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711114269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711130387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711706405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.71175019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.711782716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712287263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712329828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712360234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712382771Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712407208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712431035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712454062Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712478629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[0]R flush_numbers[0..12]: [53, 52, 50, 51, 51, 47, 48, 50, 52, 55, 55, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712501906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[0]R generation[0..12]: [34, 33, 32, 32, 32, 29, 30, 32, 33, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712525973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.71255223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[1]R flush_numbers[0..12]: [53, 52, 50, 51, 51, 47, 48, 50, 52, 55, 55, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712579677Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[1]R generation[0..12]: [34, 33, 32, 32, 32, 29, 30, 32, 33, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712605843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712639389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[2]R flush_numbers[0..12]: [53, 52, 50, 51, 51, 47, 48, 50, 52, 55, 55, 55]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:41:50.712670515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[2]R generation[0..12]: [34, 33, 32, 32, 32, 29, 30, 32, 33, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712700212Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712730498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712758864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712787751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712816607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712845364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"":"downstairs","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.71287409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712904246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"0","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712927174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"1","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712941382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"client":"2","":"downstairs","session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.71295519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712968248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"0b0629ee-4844-4480-a573-93b7615247bb is now active with session: 4a3678ca-9dd6-4102-a2d8-7caade64f5c4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712981387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.712997445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326,"session_id":"4a3678ca-9dd6-4102-a2d8-7caade64f5c4"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:50.713010293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 36864 data: 7 6 6 5 5 4 3 3 4 02/30 Write block 30 len 24576 data: 7 8 8 7 7 6 03/30 Read block 90 len 20480 04/30 Read block 567 len 28672 05/30 Flush 06/30 Write block 6 len 40960 data: 5 6 6 5 5 5 6 6 8 6 07/30 Write block 505 len 12288 data: 5 6 5 08/30 Write block 141 len 12288 data: 3 2 2 09/30 Write block 202 len 40960 data: 5 5 4 5 4 5 4 3 4 7 10/30 Read block 204 len 8192 11/30 Write block 339 len 24576 data: 2 2 2 2 2 3 12/30 Read block 441 len 36864 13/30 Flush 14/30 Write block 531 len 36864 data: 2 2 2 2 3 3 3 3 4 15/30 Read block 372 len 12288 16/30 Read block 375 len 12288 17/30 Read block 116 len 24576 18/30 Write block 198 len 4096 data: 5 19/30 Read block 231 len 24576 20/30 Write block 446 len 8192 data: 5 5 21/30 Read block 304 len 28672 22/30 Write block 2 len 4096 data: 3 23/30 Flush 24/30 Write block 48 len 24576 data: 5 3 3 3 3 5 25/30 Write block 366 len 32768 data: 3 3 3 4 3 4 4 3 26/30 Read block 86 len 16384 27/30 Write block 378 len 24576 data: 3 3 3 3 4 4 28/30 Write block 590 len 24576 data: 5 7 6 7 7 5 29/30 Read block 354 len 12288 30/30 Write block 287 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:50.731875089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 58 58 58 F F F 1 020-039 36 36 36 56 56 56 F F F 2 040-059 36 36 36 56 56 56 T T T 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 36 36 36 57 57 57 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 26 26 26 44 44 44 T T T 15 300-319 35 35 35 55 55 55 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 30 30 30 48 48 48 T T T 19 380-399 19 19 19 38 38 38 T T T 20 400-419 35 35 35 55 55 55 F F F 21 420-439 34 34 34 54 54 54 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 34 34 34 53 53 53 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 36 36 36 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 32 32 32 50 50 50 T T T Max gen: 36, Max flush: 58 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 58 58 58 F F F 1 020-039 36 36 36 56 56 56 F F F 2 040-059 37 37 37 59 59 59 F F F 3 060-079 32 32 32 51 51 51 F F F 4 080-099 32 32 32 51 51 51 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 36 36 36 57 57 57 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 37 37 37 59 59 59 F F F 15 300-319 35 35 35 55 55 55 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 35 35 35 55 55 55 F F F 21 420-439 34 34 34 54 54 54 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 34 34 34 53 53 53 F F F 24 480-499 29 29 29 47 47 47 F F F 25 500-519 36 36 36 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 37, Max flush: 59 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:54.97439931Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02083288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.021512775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02155664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.021572288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.021591935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} {"msg":"Crucible 1b953d63-3b4b-4231-aedc-c1c9180f1226 has session id: 21c436e5-18ae-49cb-8b95-aed339baa532","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022180142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Upstairs opts: Upstairs UUID: 1b953d63-3b4b-4231-aedc-c1c9180f1226, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022223537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Crucible stats registered with UUID: 1b953d63-3b4b-4231-aedc-c1c9180f1226","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022255263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:55.02228054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022714276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022740853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} {"msg":"1b953d63-3b4b-4231-aedc-c1c9180f1226 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022793536Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022822722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022847409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022862438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022876416Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022889844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.022968764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023023827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023065012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023233661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_connection connected from Ok(127.0.0.1:63343)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023273936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_connection connected from Ok(127.0.0.1:43740)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023304283Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_connection connected from Ok(127.0.0.1:44215)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023332529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023769775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023841316Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02388479Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.023913827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.024170595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02421281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.024473807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.024657644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02485119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025046266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025089111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025108658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025122357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025136425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025150043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025331041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025354138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[0]R flush_numbers[0..12]: [58, 56, 59, 51, 51, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025368776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[0]R generation[0..12]: [36, 36, 37, 32, 32, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025383054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025397423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[1]R flush_numbers[0..12]: [58, 56, 59, 51, 51, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025411431Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[1]R generation[0..12]: [36, 36, 37, 32, 32, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025425119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025439097Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[2]R flush_numbers[0..12]: [58, 56, 59, 51, 51, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025452676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[2]R generation[0..12]: [36, 36, 37, 32, 32, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025466294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02549561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025523447Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025549504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025564142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025585499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025617685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025643942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"0","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.02566136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"1","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025675128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"client":"2","":"downstairs","session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025693386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025717533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"1b953d63-3b4b-4231-aedc-c1c9180f1226 is now active with session: 21c436e5-18ae-49cb-8b95-aed339baa532","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025744329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025764477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334,"session_id":"21c436e5-18ae-49cb-8b95-aed339baa532"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:55.025777405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 364 len 36864 data: 4 3 4 4 4 5 4 5 5 03/30 Write block 507 len 28672 data: 6 5 4 4 3 3 4 04/30 Write block 483 len 12288 data: 7 5 5 05/30 Flush 06/30 Flush 07/30 Write block 59 len 24576 data: 7 5 7 7 5 4 08/30 Read block 271 len 12288 09/30 Read block 288 len 12288 10/30 Read block 222 len 8192 11/30 Write block 421 len 8192 data: 6 5 12/30 Read block 77 len 40960 13/30 Read block 529 len 28672 14/30 Read block 399 len 28672 15/30 Read block 598 len 4096 16/30 Write block 300 len 24576 data: 4 4 4 4 4 3 17/30 Read block 382 len 28672 18/30 Flush 19/30 Write block 14 len 28672 data: 9 7 5 4 6 8 9 20/30 Write block 521 len 8192 data: 4 3 21/30 Read block 47 len 28672 22/30 Read block 173 len 12288 23/30 Read block 108 len 4096 24/30 Read block 242 len 20480 25/30 Read block 96 len 40960 26/30 Read block 591 len 12288 27/30 Read block 4 len 4096 28/30 Read block 443 len 28672 29/30 Write block 473 len 40960 data: 4 3 3 3 5 5 5 5 6 6 30/30 Write block 86 len 16384 data: 7 5 6 6 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:55.044954912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 58 58 58 T T T 1 020-039 36 36 36 56 56 56 T T T 2 040-059 38 38 38 63 63 63 F F F 3 060-079 38 38 38 63 63 63 F F F 4 080-099 32 32 32 51 51 51 T T T 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 36 36 36 57 57 57 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 37 37 37 59 59 59 F F F 15 300-319 38 38 38 63 63 63 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 38 38 38 61 61 61 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 35 35 35 55 55 55 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 34 34 34 53 53 53 T T T 24 480-499 38 38 38 61 61 61 T T T 25 500-519 38 38 38 61 61 61 F F F 26 520-539 36 36 36 58 58 58 T T T 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 38, Max flush: 63 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 39 39 39 64 64 64 F F F 1 020-039 39 39 39 64 64 64 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 38 38 38 63 63 63 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 29 29 29 47 47 47 F F F 6 120-139 30 30 30 48 48 48 F F F 7 140-159 36 36 36 57 57 57 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 36 36 36 58 58 58 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 16 16 16 33 33 33 F F F 14 280-299 37 37 37 59 59 59 F F F 15 300-319 38 38 38 63 63 63 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 38 38 38 61 61 61 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 35 35 35 55 55 55 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 39 39 39 64 64 64 F F F 24 480-499 39 39 39 64 64 64 F F F 25 500-519 38 38 38 61 61 61 F F F 26 520-539 39 39 39 64 64 64 F F F 27 540-559 32 32 32 50 50 50 F F F 28 560-579 32 32 32 50 50 50 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 39, Max flush: 64 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.292589379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.337837565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.338561366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.33860597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.338632247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.338660083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} {"msg":"Crucible 8cf161da-f800-457d-ba8c-f00349b9b85a has session id: 96a95d0b-3f18-42bd-8995-c518c234cb89","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339189388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Upstairs opts: Upstairs UUID: 8cf161da-f800-457d-ba8c-f00349b9b85a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339234802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Crucible stats registered with UUID: 8cf161da-f800-457d-ba8c-f00349b9b85a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339270547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:59.339297584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339833628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339870933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} {"msg":"8cf161da-f800-457d-ba8c-f00349b9b85a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339926686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.339963431Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340003796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340036342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340065439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340098535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340129311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340161887Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340199512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340234608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_connection connected from Ok(127.0.0.1:56282)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340262834Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_connection connected from Ok(127.0.0.1:55900)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340349153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_connection connected from Ok(127.0.0.1:58075)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340395618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340950769Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.340985684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341012451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341040498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341174921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341220845Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341687407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341731512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.341992199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342274684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.34230363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342320028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342348805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342368173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342382911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342444633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.34246491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 63, 63, 64, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342479069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[0]R generation[0..12]: [39, 39, 38, 38, 39, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342494177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342508655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 63, 63, 64, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342523203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[1]R generation[0..12]: [39, 39, 38, 38, 39, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342536932Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.34255087Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 63, 63, 64, 47, 48, 57, 52, 58, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342565438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[2]R generation[0..12]: [39, 39, 38, 38, 39, 29, 30, 36, 33, 36, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342579326Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342594125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342608233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342625311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342639219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342699092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"":"downstairs","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342728858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342743146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"0","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342759394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"1","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342773232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"client":"2","":"downstairs","session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342786741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342799279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"8cf161da-f800-457d-ba8c-f00349b9b85a is now active with session: 96a95d0b-3f18-42bd-8995-c518c234cb89","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342812437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342841054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342,"session_id":"96a95d0b-3f18-42bd-8995-c518c234cb89"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:41:59.342854242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 8192 data: 4 5 02/30 Read block 6 len 12288 03/30 Read block 142 len 8192 04/30 Write block 366 len 24576 data: 5 5 5 6 5 6 05/30 Write block 155 len 28672 data: 5 5 4 3 2 3 3 06/30 Read block 335 len 24576 07/30 Write block 8 len 32768 data: 7 6 6 6 7 7 10 8 08/30 Read block 488 len 28672 09/30 Read block 158 len 28672 10/30 Write block 274 len 8192 data: 2 3 11/30 Read block 394 len 24576 12/30 Write block 503 len 40960 data: 6 6 6 7 7 6 5 5 4 4 13/30 Write block 48 len 16384 data: 6 4 4 4 14/30 Write block 396 len 16384 data: 5 6 6 7 15/30 Read block 264 len 40960 16/30 Read block 264 len 24576 17/30 Flush 18/30 Write block 133 len 16384 data: 4 4 4 3 19/30 Write block 3 len 36864 data: 3 3 4 6 7 8 7 7 7 20/30 Flush 21/30 Write block 555 len 12288 data: 3 4 5 22/30 Flush 23/30 Write block 102 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Write block 567 len 8192 data: 4 6 25/30 Read block 60 len 20480 26/30 Write block 412 len 28672 data: 2 3 3 3 3 4 4 27/30 Write block 293 len 40960 data: 2 2 2 2 2 2 4 5 5 5 28/30 Read block 241 len 40960 29/30 Write block 50 len 16384 data: 5 5 4 6 30/30 Read block 119 len 8192 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 6 6 8 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:41:59.362608667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 66 66 66 F F F 1 020-039 39 39 39 64 64 64 F F F 2 040-059 40 40 40 65 65 65 T T T 3 060-079 38 38 38 63 63 63 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 29 29 29 47 47 47 T T T 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 40 40 40 65 65 65 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 37 37 37 59 59 59 T T T 15 300-319 38 38 38 63 63 63 T T T 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 40 40 40 65 65 65 F F F 20 400-419 35 35 35 55 55 55 T T T 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 39 39 39 64 64 64 F F F 24 480-499 39 39 39 64 64 64 F F F 25 500-519 40 40 40 65 65 65 F F F 26 520-539 39 39 39 64 64 64 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 32 32 32 50 50 50 T T T 29 580-599 37 37 37 59 59 59 F F F Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 66 66 66 F F F 1 020-039 39 39 39 64 64 64 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 38 38 38 63 63 63 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 40 40 40 65 65 65 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 40 40 40 65 65 65 F F F 19 380-399 40 40 40 65 65 65 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 58 58 58 F F F 23 460-479 39 39 39 64 64 64 F F F 24 480-499 39 39 39 64 64 64 F F F 25 500-519 40 40 40 65 65 65 F F F 26 520-539 39 39 39 64 64 64 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.607516123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:03.650757989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.651220361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.683234693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.683920058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.683956683Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.683971251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.68398628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} {"msg":"Crucible c1a08635-cc4c-4d09-970f-bf3614c40bb6 has session id: b276b162-1adb-4fcf-a390-545a1e552df7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.684438144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Upstairs opts: Upstairs UUID: c1a08635-cc4c-4d09-970f-bf3614c40bb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.684480698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Crucible stats registered with UUID: c1a08635-cc4c-4d09-970f-bf3614c40bb6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.684506935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:03.684521863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.684969307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.684994684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} {"msg":"c1a08635-cc4c-4d09-970f-bf3614c40bb6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685055847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685074674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685098271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.68511254Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685128338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685153125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685180061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685208448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685230275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685258531Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63701)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685484834Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62000)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685565104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44498)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.685603019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686177917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686229271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686251358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686266976Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686281484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686295643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686865412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686908286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.686944292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687456618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687498093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.68752896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687598081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687626817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687646665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687660193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687675221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[0]R flush_numbers[0..12]: [66, 64, 68, 63, 64, 68, 66, 65, 65, 65, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687689429Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[0]R generation[0..12]: [40, 39, 41, 38, 39, 41, 40, 40, 40, 40, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687704418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687718856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[1]R flush_numbers[0..12]: [66, 64, 68, 63, 64, 68, 66, 65, 65, 65, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687732714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[1]R generation[0..12]: [40, 39, 41, 38, 39, 41, 40, 40, 40, 40, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687746422Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687760601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[2]R flush_numbers[0..12]: [66, 64, 68, 63, 64, 68, 66, 65, 65, 65, 57, 55]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687774299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[2]R generation[0..12]: [40, 39, 41, 38, 39, 41, 40, 40, 40, 40, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687789007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687803245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687816224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687829792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687867957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"":"downstairs","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687882545Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687896014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"0","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687910452Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"1","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.68792395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"client":"2","":"downstairs","session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687937639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687950917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"c1a08635-cc4c-4d09-970f-bf3614c40bb6 is now active with session: b276b162-1adb-4fcf-a390-545a1e552df7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687964165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687976534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350,"session_id":"b276b162-1adb-4fcf-a390-545a1e552df7"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:03.687989802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 576 len 8192 data: 4 5 02/30 Write block 450 len 32768 data: 3 3 3 3 4 3 3 3 03/30 Read block 482 len 4096 04/30 Write block 11 len 20480 data: 8 8 8 11 9 05/30 Read block 189 len 36864 06/30 Read block 126 len 16384 07/30 Write block 395 len 8192 data: 5 6 08/30 Read block 187 len 24576 09/30 Flush 10/30 Flush 11/30 Write block 518 len 40960 data: 5 5 4 5 4 4 3 2 2 2 12/30 Write block 386 len 4096 data: 3 13/30 Read block 240 len 4096 14/30 Write block 7 len 12288 data: 8 9 8 15/30 Read block 544 len 40960 16/30 Write block 369 len 40960 data: 7 6 7 6 4 2 2 3 4 4 17/30 Write block 154 len 16384 data: 6 6 6 5 18/30 Flush 19/30 Write block 101 len 24576 data: 2 3 3 3 3 3 20/30 Write block 322 len 24576 data: 6 5 4 3 4 4 21/30 Read block 439 len 36864 22/30 Write block 413 len 24576 data: 4 4 4 4 5 5 23/30 Write block 386 len 16384 data: 4 3 2 2 24/30 Read block 362 len 32768 25/30 Write block 59 len 16384 data: 8 6 8 8 26/30 Read block 425 len 4096 27/30 Write block 226 len 24576 data: 6 5 5 6 6 7 28/30 Read block 2 len 36864 29/30 Read block 10 len 4096 30/30 Write block 508 len 32768 data: 7 6 6 5 5 5 5 6 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:03.707248049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 39 39 39 64 64 64 F F F 2 040-059 41 41 41 68 68 68 T T T 3 060-079 38 38 38 63 63 63 T T T 4 080-099 39 39 39 64 64 64 F F F 5 100-119 41 41 41 68 68 68 T T T 6 120-139 40 40 40 66 66 66 F F F 7 140-159 42 42 42 71 71 71 F F F 8 160-179 40 40 40 65 65 65 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 35 35 35 55 55 55 T T T 12 240-259 35 35 35 55 55 55 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 36 36 36 57 57 57 T T T 17 340-359 36 36 36 57 57 57 F F F 18 360-379 42 42 42 71 71 71 F F F 19 380-399 42 42 42 71 71 71 T T T 20 400-419 41 41 41 68 68 68 T T T 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 39 39 39 64 64 64 F F F 24 480-499 39 39 39 64 64 64 F F F 25 500-519 42 42 42 71 71 71 T T T 26 520-539 42 42 42 71 71 71 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 69 69 69 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 42, Max flush: 71 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 71 71 71 F F F 1 020-039 39 39 39 64 64 64 F F F 2 040-059 43 43 43 72 72 72 F F F 3 060-079 43 43 43 72 72 72 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 43 43 43 72 72 72 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 42 42 42 71 71 71 F F F 8 160-179 40 40 40 65 65 65 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 36 36 36 57 57 57 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 35 35 35 55 55 55 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 36 36 36 57 57 57 F F F 18 360-379 42 42 42 71 71 71 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 43 43 43 72 72 72 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 39 39 39 64 64 64 F F F 24 480-499 39 39 39 64 64 64 F F F 25 500-519 43 43 43 72 72 72 F F F 26 520-539 42 42 42 71 71 71 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 69 69 69 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 43, Max flush: 72 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:07.989703158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:08.03177007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.032183359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.063063992Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.063684264Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.06372131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.063736508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.063752246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} {"msg":"Crucible 5c3cbee0-f900-45c3-aeb7-8a1ad3e8f508 has session id: adc48047-4cdb-42f3-b91d-2d41653ad511","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.06420346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Upstairs opts: Upstairs UUID: 5c3cbee0-f900-45c3-aeb7-8a1ad3e8f508, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064246725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Crucible stats registered with UUID: 5c3cbee0-f900-45c3-aeb7-8a1ad3e8f508","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064276761Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:08.064292859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064725655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064750312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} {"msg":"5c3cbee0-f900-45c3-aeb7-8a1ad3e8f508 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064803765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064833152Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064860948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064888825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064917481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064943038Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064970655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.064999901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065028737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065055914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_connection connected from Ok(127.0.0.1:63678)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.06516478Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_connection connected from Ok(127.0.0.1:36824)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065200506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_connection connected from Ok(127.0.0.1:57295)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065233052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065805101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065833897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065849075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065863054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065927926Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.065948403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.066478617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.06661823Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.066648466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067105259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067133066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067148574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067220485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067241513Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067255441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067272249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067287027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[0]R flush_numbers[0..12]: [71, 64, 72, 72, 64, 72, 66, 71, 65, 65, 57, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067301825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[0]R generation[0..12]: [42, 39, 43, 43, 39, 43, 40, 42, 40, 40, 36, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067315863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067331151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[1]R flush_numbers[0..12]: [71, 64, 72, 72, 64, 72, 66, 71, 65, 65, 57, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.0673452Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[1]R generation[0..12]: [42, 39, 43, 43, 39, 43, 40, 42, 40, 40, 36, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067359738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067374106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[2]R flush_numbers[0..12]: [71, 64, 72, 72, 64, 72, 66, 71, 65, 65, 57, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067387854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[2]R generation[0..12]: [42, 39, 43, 43, 39, 43, 40, 42, 40, 40, 36, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067402662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067416901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067430299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067447807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067466065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"All extents match","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067494981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"":"downstairs","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067511299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067524217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"0","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067538556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"1","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067553274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"client":"2","":"downstairs","session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067567202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067579791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"5c3cbee0-f900-45c3-aeb7-8a1ad3e8f508 is now active with session: adc48047-4cdb-42f3-b91d-2d41653ad511","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067592759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067605637Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358,"session_id":"adc48047-4cdb-42f3-b91d-2d41653ad511"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:08.067618236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 32768 data: 3 3 5 6 5 6 6 4 02/30 Write block 100 len 32768 data: 2 3 4 4 4 4 4 3 03/30 Write block 249 len 36864 data: 2 2 2 2 2 2 2 4 4 04/30 Write block 355 len 8192 data: 3 3 05/30 Write block 156 len 40960 data: 7 6 4 3 4 4 3 3 6 6 06/30 Write block 38 len 32768 data: 4 4 5 3 3 4 4 4 07/30 Write block 12 len 24576 data: 9 9 12 10 6 5 08/30 Read block 186 len 12288 09/30 Write block 248 len 12288 data: 3 3 3 10/30 Write block 473 len 36864 data: 5 4 4 4 6 6 6 6 7 11/30 Flush 12/30 Write block 115 len 4096 data: 4 13/30 Read block 193 len 12288 14/30 Write block 506 len 24576 data: 8 8 8 7 7 6 15/30 Read block 47 len 20480 16/30 Read block 301 len 36864 17/30 Flush 18/30 Write block 576 len 12288 data: 5 6 4 19/30 Flush 20/30 Write block 207 len 28672 data: 6 5 4 5 8 6 6 21/30 Read block 564 len 32768 22/30 Write block 116 len 12288 data: 4 3 3 23/30 Write block 209 len 36864 data: 5 6 9 7 7 5 4 4 3 24/30 Write block 543 len 4096 data: 4 25/30 Read block 210 len 28672 26/30 Flush 27/30 Write block 22 len 40960 data: 10 10 9 10 8 9 8 8 8 9 28/30 Write block 531 len 12288 data: 3 3 3 29/30 Write block 288 len 12288 data: 7 5 5 30/30 Read block 201 len 16384 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:08.086188068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 T T T 2 040-059 44 44 44 73 73 73 F F F 3 060-079 43 43 43 72 72 72 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 73 73 73 F F F 8 160-179 44 44 44 73 73 73 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 44 44 44 73 73 73 T T T 15 300-319 41 41 41 68 68 68 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 44 44 44 73 73 73 F F F 18 360-379 42 42 42 71 71 71 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 43 43 43 72 72 72 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 44 44 44 73 73 73 F F F 25 500-519 44 44 44 74 74 74 F F F 26 520-539 42 42 42 71 71 71 T T T 27 540-559 44 44 44 76 76 76 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 44, Max flush: 76 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 45 45 45 77 77 77 F F F 2 040-059 44 44 44 73 73 73 F F F 3 060-079 43 43 43 72 72 72 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 73 73 73 F F F 8 160-179 44 44 44 73 73 73 F F F 9 180-199 40 40 40 65 65 65 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 40 40 40 65 65 65 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 44 44 44 73 73 73 F F F 18 360-379 42 42 42 71 71 71 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 43 43 43 72 72 72 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 44 44 44 73 73 73 F F F 25 500-519 44 44 44 74 74 74 F F F 26 520-539 45 45 45 77 77 77 F F F 27 540-559 44 44 44 76 76 76 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.344035356Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.388516998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389295661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389343725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389361193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389377041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} {"msg":"Crucible bd9c9a1d-cd7f-4ce2-b4a6-581efa8a06a6 has session id: f6917946-9acd-4ffd-927a-e5751a9abfc0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389735646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Upstairs opts: Upstairs UUID: bd9c9a1d-cd7f-4ce2-b4a6-581efa8a06a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389777741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Crucible stats registered with UUID: bd9c9a1d-cd7f-4ce2-b4a6-581efa8a06a6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.389803488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:12.389826965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390354489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390389975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} {"msg":"bd9c9a1d-cd7f-4ce2-b4a6-581efa8a06a6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39042965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390457457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390493232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390523808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390542956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390567933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.3905908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390607458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390660691Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390698187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47998)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.390728873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:51013)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39075663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49821)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391038615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391527394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39155593Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391571128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391589606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391603604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.391617023Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392266562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392306467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392338713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392870937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392898873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392913642Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39292748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392941408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392968945Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392983133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.392996651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[0]R flush_numbers[0..12]: [73, 77, 73, 72, 64, 76, 66, 73, 73, 65, 76, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39301055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[0]R generation[0..12]: [44, 45, 44, 43, 39, 44, 40, 44, 44, 40, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393028147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393042735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[1]R flush_numbers[0..12]: [73, 77, 73, 72, 64, 76, 66, 73, 73, 65, 76, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393056414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[1]R generation[0..12]: [44, 45, 44, 43, 39, 44, 40, 44, 44, 40, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393070052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.39308414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[2]R flush_numbers[0..12]: [73, 77, 73, 72, 64, 76, 66, 73, 73, 65, 76, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393097759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[2]R generation[0..12]: [44, 45, 44, 43, 39, 44, 40, 44, 44, 40, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393111647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393125695Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393138723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393152032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393178179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"":"downstairs","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393191577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393203766Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"0","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393223723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"1","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393239381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"client":"2","":"downstairs","session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393252959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393265428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"bd9c9a1d-cd7f-4ce2-b4a6-581efa8a06a6 is now active with session: f6917946-9acd-4ffd-927a-e5751a9abfc0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393279046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393291224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366,"session_id":"f6917946-9acd-4ffd-927a-e5751a9abfc0"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:12.393303573Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 278 len 8192 data: 2 2 02/30 Read block 63 len 32768 03/30 Read block 361 len 40960 04/30 Write block 199 len 16384 data: 4 4 5 6 05/30 Write block 324 len 28672 data: 5 4 5 5 4 3 3 06/30 Write block 495 len 40960 data: 5 4 4 4 5 6 5 6 7 7 07/30 Read block 133 len 16384 08/30 Write block 531 len 20480 data: 4 4 4 3 4 09/30 Write block 406 len 4096 data: 5 10/30 Read block 88 len 24576 11/30 Write block 212 len 8192 data: 8 8 12/30 Write block 497 len 28672 data: 5 5 6 7 6 7 8 13/30 Read block 332 len 40960 14/30 Write block 288 len 8192 data: 8 6 15/30 Read block 197 len 24576 16/30 Write block 43 len 20480 data: 5 5 5 4 4 17/30 Read block 507 len 24576 18/30 Write block 411 len 12288 data: 3 3 5 19/30 Read block 345 len 4096 20/30 Write block 167 len 32768 data: 5 5 4 4 4 2 2 4 21/30 Write block 355 len 24576 data: 4 4 2 2 3 3 22/30 Read block 252 len 28672 23/30 Flush 24/30 Write block 288 len 8192 data: 9 7 25/30 Read block 214 len 4096 26/30 Read block 160 len 40960 27/30 Read block 3 len 8192 28/30 Read block 254 len 8192 29/30 Write block 364 len 16384 data: 5 4 6 6 30/30 Write block 100 len 28672 data: 3 4 5 5 5 5 5 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:12.411849908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 45 45 45 77 77 77 F F F 2 040-059 46 46 46 78 78 78 F F F 3 060-079 43 43 43 72 72 72 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 44 44 44 76 76 76 T T T 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 73 73 73 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 46 46 46 78 78 78 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 46 46 46 78 78 78 F F F 14 280-299 46 46 46 78 78 78 T T T 15 300-319 41 41 41 68 68 68 F F F 16 320-339 46 46 46 78 78 78 F F F 17 340-359 46 46 46 78 78 78 F F F 18 360-379 46 46 46 78 78 78 T T T 19 380-399 43 43 43 72 72 72 F F F 20 400-419 46 46 46 78 78 78 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 46 46 46 78 78 78 F F F 27 540-559 44 44 44 76 76 76 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 46, Max flush: 78 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 45 45 45 77 77 77 F F F 2 040-059 46 46 46 78 78 78 F F F 3 060-079 43 43 43 72 72 72 F F F 4 080-099 39 39 39 64 64 64 F F F 5 100-119 47 47 47 79 79 79 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 73 73 73 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 46 46 46 78 78 78 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 46 46 46 78 78 78 F F F 14 280-299 47 47 47 79 79 79 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 46 46 46 78 78 78 F F F 17 340-359 46 46 46 78 78 78 F F F 18 360-379 47 47 47 79 79 79 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 46 46 46 78 78 78 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 46 46 46 78 78 78 F F F 27 540-559 44 44 44 76 76 76 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 47, Max flush: 79 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.637020586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:16.679849724Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.680276741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.711650992Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.712343076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.712380682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71239499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.712409618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} {"msg":"Crucible f01b6a95-8cf6-4d8a-b599-aa6214b69c0f has session id: db6f52a7-72dc-4b31-a8b0-29f76f5c0dda","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71295603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Upstairs opts: Upstairs UUID: f01b6a95-8cf6-4d8a-b599-aa6214b69c0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.712998625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Crucible stats registered with UUID: f01b6a95-8cf6-4d8a-b599-aa6214b69c0f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713025071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:16.7130406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713495203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.7135195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} {"msg":"f01b6a95-8cf6-4d8a-b599-aa6214b69c0f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71359917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713616908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713643565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713658663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71367956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713707146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713734453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71376136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713790496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713820572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_connection connected from Ok(127.0.0.1:47062)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713851409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_connection connected from Ok(127.0.0.1:49002)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713877176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_connection connected from Ok(127.0.0.1:56428)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.713927459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.714548542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.714585777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.714615424Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.71464642Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.714703613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.714790012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715247515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715276602Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715292509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715886446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715927021Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715957507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.715982984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716007581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716032378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716054525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716078312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[0]R flush_numbers[0..12]: [73, 77, 78, 72, 64, 79, 66, 73, 78, 78, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716103479Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[0]R generation[0..12]: [44, 45, 46, 43, 39, 47, 40, 44, 46, 46, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716128576Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716153562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[1]R flush_numbers[0..12]: [73, 77, 78, 72, 64, 79, 66, 73, 78, 78, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716182419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[1]R generation[0..12]: [44, 45, 46, 43, 39, 47, 40, 44, 46, 46, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716209866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716237942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[2]R flush_numbers[0..12]: [73, 77, 78, 72, 64, 79, 66, 73, 78, 78, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716265059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[2]R generation[0..12]: [44, 45, 46, 43, 39, 47, 40, 44, 46, 46, 46, 43]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-11T23:42:16.716299244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716330361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716374475Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716403502Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716432578Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716460675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"":"downstairs","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716490601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716516648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"0","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716549923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"1","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716570541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"client":"2","":"downstairs","session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716588188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716601307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"f01b6a95-8cf6-4d8a-b599-aa6214b69c0f is now active with session: db6f52a7-72dc-4b31-a8b0-29f76f5c0dda","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716614385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716626854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374,"session_id":"db6f52a7-72dc-4b31-a8b0-29f76f5c0dda"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:16.716639372Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 424 len 24576 data: 4 4 3 3 3 3 02/30 Write block 15 len 16384 data: 11 7 6 7 03/30 Read block 406 len 24576 04/30 Read block 284 len 24576 05/30 Read block 118 len 12288 06/30 Read block 462 len 20480 07/30 Read block 99 len 12288 08/30 Read block 352 len 32768 09/30 Read block 114 len 8192 10/30 Write block 97 len 20480 data: 3 3 2 4 5 11/30 Write block 273 len 16384 data: 2 3 4 3 12/30 Read block 45 len 28672 13/30 Write block 249 len 8192 data: 4 4 14/30 Write block 526 len 16384 data: 3 3 2 2 15/30 Read block 431 len 8192 16/30 Write block 74 len 4096 data: 5 17/30 Read block 501 len 20480 18/30 Flush 19/30 Write block 543 len 12288 data: 5 4 3 20/30 Write block 389 len 28672 data: 3 2 3 3 3 4 6 21/30 Read block 316 len 36864 22/30 Read block 298 len 32768 23/30 Read block 452 len 8192 24/30 Read block 305 len 4096 25/30 Write block 273 len 36864 data: 3 4 5 4 3 3 3 2 3 26/30 Read block 321 len 24576 27/30 Write block 156 len 12288 data: 8 7 5 28/30 Write block 11 len 8192 data: 9 10 29/30 Write block 552 len 16384 data: 4 4 4 4 30/30 Write block 182 len 12288 data: 7 8 8 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:16.734425782Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 80 80 80 T T T 1 020-039 45 45 45 77 77 77 F F F 2 040-059 46 46 46 78 78 78 F F F 3 060-079 48 48 48 80 80 80 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 73 73 73 T T T 8 160-179 46 46 46 78 78 78 F F F 9 180-199 46 46 46 78 78 78 T T T 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 48 48 48 80 80 80 T T T 14 280-299 47 47 47 79 79 79 T T T 15 300-319 41 41 41 68 68 68 F F F 16 320-339 46 46 46 78 78 78 F F F 17 340-359 46 46 46 78 78 78 F F F 18 360-379 47 47 47 79 79 79 F F F 19 380-399 43 43 43 72 72 72 T T T 20 400-419 46 46 46 78 78 78 F F F 21 420-439 48 48 48 80 80 80 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 44 44 44 76 76 76 T T T 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 48, Max flush: 80 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 45 45 45 77 77 77 F F F 2 040-059 46 46 46 78 78 78 F F F 3 060-079 48 48 48 80 80 80 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 49 49 49 81 81 81 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 49 49 49 81 81 81 F F F 14 280-299 49 49 49 81 81 81 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 46 46 46 78 78 78 F F F 17 340-359 46 46 46 78 78 78 F F F 18 360-379 47 47 47 79 79 79 F F F 19 380-399 49 49 49 81 81 81 F F F 20 400-419 46 46 46 78 78 78 F F F 21 420-439 48 48 48 80 80 80 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 44 44 44 73 73 73 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 49, Max flush: 81 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.020295756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:21.062357719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.06275453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.094115233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095001062Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095042308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095057066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095071944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} {"msg":"Crucible 3087a4bb-8c86-475f-9c71-290fc33ce7b7 has session id: 5bbd24f3-1875-4e12-bd1b-75cb0672f1bf","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095559763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Upstairs opts: Upstairs UUID: 3087a4bb-8c86-475f-9c71-290fc33ce7b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095602408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Crucible stats registered with UUID: 3087a4bb-8c86-475f-9c71-290fc33ce7b7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.095629435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:21.095644953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096074109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096099346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} {"msg":"3087a4bb-8c86-475f-9c71-290fc33ce7b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096157489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096176556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096200624Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096219911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096234369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096248108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096261766Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096276044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096296892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096322808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:65411)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096349735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:51470)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096383521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:42899)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.096411517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097150775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.09719365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097226076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097253303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097269001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097288168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097767869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.097891863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.09792237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098383462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098412249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098426717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098505107Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098526355Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098540493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098552941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.09856629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[0]R flush_numbers[0..12]: [81, 77, 78, 80, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098580338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[0]R generation[0..12]: [49, 45, 46, 48, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098595576Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098609914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[1]R flush_numbers[0..12]: [81, 77, 78, 80, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098627932Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[1]R generation[0..12]: [49, 45, 46, 48, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.09864202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098656428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[2]R flush_numbers[0..12]: [81, 77, 78, 80, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098670367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[2]R generation[0..12]: [49, 45, 46, 48, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098684145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098698333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098711972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.09872524Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098738378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098751587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"":"downstairs","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098764935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098777323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"0","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-12-11T23:42:21.098798821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"1","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098812389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"client":"2","":"downstairs","session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098825857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098840776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"3087a4bb-8c86-475f-9c71-290fc33ce7b7 is now active with session: 5bbd24f3-1875-4e12-bd1b-75cb0672f1bf","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098854194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098866592Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382,"session_id":"5bbd24f3-1875-4e12-bd1b-75cb0672f1bf"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:21.098879051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 28672 data: 4 4 6 5 5 5 6 02/30 Read block 124 len 32768 03/30 Read block 521 len 16384 04/30 Read block 103 len 4096 05/30 Read block 404 len 32768 06/30 Read block 348 len 40960 07/30 Read block 270 len 28672 08/30 Write block 459 len 8192 data: 3 3 09/30 Flush 10/30 Write block 527 len 4096 data: 4 11/30 Read block 430 len 8192 12/30 Read block 299 len 36864 13/30 Read block 199 len 20480 14/30 Write block 368 len 40960 data: 6 8 7 8 7 5 3 3 4 5 15/30 Write block 511 len 32768 data: 7 6 6 6 7 5 6 6 16/30 Write block 530 len 8192 data: 2 5 17/30 Read block 277 len 8192 18/30 Write block 76 len 12288 data: 4 4 5 19/30 Write block 303 len 4096 data: 5 20/30 Write block 461 len 24576 data: 3 2 2 2 3 4 21/30 Read block 526 len 20480 22/30 Read block 186 len 28672 23/30 Write block 325 len 32768 data: 5 6 6 5 4 4 2 2 24/30 Write block 39 len 4096 data: 5 25/30 Write block 493 len 28672 data: 3 3 6 5 6 6 7 26/30 Write block 273 len 4096 data: 4 27/30 Write block 464 len 36864 data: 3 4 5 4 5 5 4 4 6 28/30 Write block 333 len 32768 data: 2 2 2 2 2 2 3 3 29/30 Read block 239 len 24576 30/30 Write block 365 len 24576 data: 5 7 7 7 9 8 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:21.117866361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 45 45 45 77 77 77 T T T 2 040-059 46 46 46 78 78 78 F F F 3 060-079 48 48 48 80 80 80 T T T 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 49 49 49 81 81 81 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 49 49 49 81 81 81 T T T 14 280-299 49 49 49 81 81 81 F F F 15 300-319 41 41 41 68 68 68 T T T 16 320-339 46 46 46 78 78 78 T T T 17 340-359 46 46 46 78 78 78 T T T 18 360-379 47 47 47 79 79 79 T T T 19 380-399 49 49 49 81 81 81 F F F 20 400-419 50 50 50 82 82 82 F F F 21 420-439 48 48 48 80 80 80 F F F 22 440-459 50 50 50 82 82 82 F F F 23 460-479 50 50 50 82 82 82 T T T 24 480-499 46 46 46 78 78 78 T T T 25 500-519 46 46 46 78 78 78 T T T 26 520-539 48 48 48 80 80 80 T T T 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 50, Max flush: 82 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 51 51 51 83 83 83 F F F 2 040-059 46 46 46 78 78 78 F F F 3 060-079 51 51 51 83 83 83 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 49 49 49 81 81 81 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 51 51 51 83 83 83 F F F 14 280-299 49 49 49 81 81 81 F F F 15 300-319 51 51 51 83 83 83 F F F 16 320-339 51 51 51 83 83 83 F F F 17 340-359 51 51 51 83 83 83 F F F 18 360-379 51 51 51 83 83 83 F F F 19 380-399 49 49 49 81 81 81 F F F 20 400-419 50 50 50 82 82 82 F F F 21 420-439 48 48 48 80 80 80 F F F 22 440-459 50 50 50 82 82 82 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 51 51 51 83 83 83 F F F 25 500-519 51 51 51 83 83 83 F F F 26 520-539 51 51 51 83 83 83 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 51, Max flush: 83 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.42358053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.468172829Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469186133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469227278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469242286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469257694Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} {"msg":"Crucible 2e8844d3-490e-4f3e-a999-6e8f3f45c36f has session id: ad1544a8-6912-4f57-8e23-0719bc812574","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469685121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Upstairs opts: Upstairs UUID: 2e8844d3-490e-4f3e-a999-6e8f3f45c36f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469732055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Crucible stats registered with UUID: 2e8844d3-490e-4f3e-a999-6e8f3f45c36f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.469758882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:25.469781569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470298165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.47033664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} {"msg":"2e8844d3-490e-4f3e-a999-6e8f3f45c36f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470404971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470436938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470467324Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470489471Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470504509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470534095Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470566282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470593638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470622995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.4706562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_connection connected from Ok(127.0.0.1:41808)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470766437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_connection connected from Ok(127.0.0.1:55442)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470871804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_connection connected from Ok(127.0.0.1:61724)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.470912428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.471683763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.471718358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.471745955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.471775561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.471900106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.47194592Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.472478044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.47251085Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.472540016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473080169Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473108256Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473123224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473135882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.47314949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473163069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473175437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473188826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[0]R flush_numbers[0..12]: [81, 83, 78, 83, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473202884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[0]R generation[0..12]: [49, 51, 46, 51, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473216772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.47323087Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[1]R flush_numbers[0..12]: [81, 83, 78, 83, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473248378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[1]R generation[0..12]: [49, 51, 46, 51, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473262367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473276595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[2]R flush_numbers[0..12]: [81, 83, 78, 83, 80, 80, 66, 81, 78, 81, 78, 72]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473290213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[2]R generation[0..12]: [49, 51, 46, 51, 48, 48, 40, 49, 46, 49, 46, 43]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473303801Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473317669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473330938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473344236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473357335Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473370403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"":"downstairs","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473383621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473401149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"0","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473414628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"1","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473427886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"client":"2","":"downstairs","session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473441314Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473455912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"2e8844d3-490e-4f3e-a999-6e8f3f45c36f is now active with session: ad1544a8-6912-4f57-8e23-0719bc812574","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473470221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473482559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390,"session_id":"ad1544a8-6912-4f57-8e23-0719bc812574"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:25.473495018Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 16384 data: 2 3 3 4 02/30 Flush 03/30 Write block 428 len 40960 data: 4 4 3 2 3 3 5 6 6 5 04/30 Read block 6 len 28672 05/30 Write block 45 len 12288 data: 6 5 5 06/30 Write block 288 len 36864 data: 10 8 6 5 2 3 3 3 3 07/30 Write block 306 len 20480 data: 4 5 5 5 5 08/30 Read block 152 len 4096 09/30 Write block 423 len 24576 data: 5 5 5 4 4 5 10/30 Write block 452 len 8192 data: 4 4 11/30 Write block 282 len 12288 data: 4 4 6 12/30 Read block 195 len 16384 13/30 Write block 388 len 4096 data: 3 14/30 Read block 266 len 8192 15/30 Write block 536 len 4096 data: 4 16/30 Read block 318 len 32768 17/30 Write block 432 len 16384 data: 4 4 6 7 18/30 Read block 153 len 16384 19/30 Read block 136 len 24576 20/30 Write block 330 len 40960 data: 5 3 3 3 3 3 3 3 3 4 21/30 Read block 158 len 8192 22/30 Read block 589 len 16384 23/30 Read block 491 len 24576 24/30 Read block 234 len 4096 25/30 Read block 145 len 16384 26/30 Flush 27/30 Write block 496 len 4096 data: 6 28/30 Write block 274 len 40960 data: 5 6 5 4 4 4 3 4 5 5 29/30 Write block 355 len 20480 data: 5 5 3 3 4 30/30 Read block 495 len 8192 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:25.492775911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 51 51 51 83 83 83 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 51 51 51 83 83 83 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 49 49 49 81 81 81 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 51 51 51 83 83 83 T T T 14 280-299 52 52 52 85 85 85 T T T 15 300-319 52 52 52 85 85 85 F F F 16 320-339 52 52 52 85 85 85 F F F 17 340-359 51 51 51 83 83 83 T T T 18 360-379 51 51 51 83 83 83 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 50 50 50 82 82 82 F F F 21 420-439 52 52 52 85 85 85 F F F 22 440-459 52 52 52 85 85 85 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 51 51 51 83 83 83 T T T 25 500-519 51 51 51 83 83 83 F F F 26 520-539 52 52 52 85 85 85 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 52, Max flush: 85 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 51 51 51 83 83 83 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 51 51 51 83 83 83 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 49 49 49 81 81 81 F F F 8 160-179 46 46 46 78 78 78 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 53 53 53 86 86 86 F F F 14 280-299 53 53 53 86 86 86 F F F 15 300-319 52 52 52 85 85 85 F F F 16 320-339 52 52 52 85 85 85 F F F 17 340-359 53 53 53 86 86 86 F F F 18 360-379 51 51 51 83 83 83 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 50 50 50 82 82 82 F F F 21 420-439 52 52 52 85 85 85 F F F 22 440-459 52 52 52 85 85 85 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 51 51 51 83 83 83 F F F 26 520-539 52 52 52 85 85 85 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 53, Max flush: 86 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.730003681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.773246868Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.773885118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.773921134Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.773941211Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.773957019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} {"msg":"Crucible 37dcfc68-cb1b-455a-8041-5db2ba20af5e has session id: 9ed99783-32fa-4559-9c0d-055af9897ec3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.774396745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Upstairs opts: Upstairs UUID: 37dcfc68-cb1b-455a-8041-5db2ba20af5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.774439029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Crucible stats registered with UUID: 37dcfc68-cb1b-455a-8041-5db2ba20af5e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.774463576Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:29.774498362Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775004149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.7750762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} {"msg":"37dcfc68-cb1b-455a-8041-5db2ba20af5e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775099237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775121075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775150201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775173908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775197275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775221102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775246229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775275605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775301472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775325359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38921)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775348986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51593)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775459252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48462)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775612843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775906207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.775931994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.776071796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.776094993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.776507282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.776658093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777117116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777158291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777187978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777256529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777286475Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777301963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777373975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777390002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.777830978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778318637Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778345614Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[0]R flush_numbers[0..12]: [81, 83, 85, 83, 80, 80, 66, 81, 78, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778360682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[0]R generation[0..12]: [49, 51, 52, 51, 48, 48, 40, 49, 46, 49, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.77837504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778389838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[1]R flush_numbers[0..12]: [81, 83, 85, 83, 80, 80, 66, 81, 78, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778403796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[1]R generation[0..12]: [49, 51, 52, 51, 48, 48, 40, 49, 46, 49, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778417695Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778432553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[2]R flush_numbers[0..12]: [81, 83, 85, 83, 80, 80, 66, 81, 78, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778446631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[2]R generation[0..12]: [49, 51, 52, 51, 48, 48, 40, 49, 46, 49, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778475658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778491885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778505514Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778518962Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.77853206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778545489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"":"downstairs","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-11T23:42:29.778558917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778578025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"0","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778591453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"1","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778604651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"client":"2","":"downstairs","session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.77861815Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778630678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"37dcfc68-cb1b-455a-8041-5db2ba20af5e is now active with session: 9ed99783-32fa-4559-9c0d-055af9897ec3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778643787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778655825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398,"session_id":"9ed99783-32fa-4559-9c0d-055af9897ec3"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:29.778668463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 29 len 32768 data: 9 9 10 9 8 8 7 6 02/30 Write block 400 len 12288 data: 5 6 6 03/30 Read block 112 len 32768 04/30 Read block 126 len 40960 05/30 Write block 151 len 28672 data: 6 7 7 7 7 9 8 06/30 Write block 299 len 28672 data: 5 6 6 6 6 5 4 07/30 Write block 413 len 20480 data: 7 6 6 6 7 08/30 Read block 431 len 28672 09/30 Flush 10/30 Flush 11/30 Write block 417 len 40960 data: 8 6 5 5 7 6 6 6 6 5 12/30 Read block 287 len 12288 13/30 Write block 359 len 36864 data: 5 4 5 6 5 6 6 8 8 14/30 Read block 517 len 24576 15/30 Read block 264 len 36864 16/30 Read block 309 len 12288 17/30 Write block 341 len 12288 data: 3 3 3 18/30 Read block 434 len 36864 19/30 Read block 512 len 12288 20/30 Write block 176 len 20480 data: 6 6 7 6 6 21/30 Write block 535 len 16384 data: 5 5 4 4 22/30 Flush 23/30 Write block 70 len 28672 data: 6 7 6 5 6 5 5 24/30 Flush 25/30 Write block 3 len 16384 data: 4 4 5 7 26/30 Write block 438 len 12288 data: 7 6 6 27/30 Read block 447 len 12288 28/30 Write block 292 len 4096 data: 3 29/30 Read block 459 len 40960 30/30 Write block 320 len 32768 data: 7 6 7 6 6 6 7 7 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:29.79889617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 T T T 1 020-039 54 54 54 87 87 87 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 54 54 54 89 89 89 F F F 9 180-199 54 54 54 89 89 89 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 53 53 53 86 86 86 F F F 14 280-299 54 54 54 87 87 87 T T T 15 300-319 54 54 54 87 87 87 F F F 16 320-339 52 52 52 85 85 85 T T T 17 340-359 54 54 54 89 89 89 F F F 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 54 54 54 89 89 89 F F F 21 420-439 54 54 54 89 89 89 T T T 22 440-459 52 52 52 85 85 85 T T T 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 51 51 51 83 83 83 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 54, Max flush: 90 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 F F F 1 020-039 54 54 54 87 87 87 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 48 48 48 80 80 80 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 54 54 54 87 87 87 F F F 8 160-179 54 54 54 89 89 89 F F F 9 180-199 54 54 54 89 89 89 F F F 10 200-219 46 46 46 78 78 78 F F F 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 53 53 53 86 86 86 F F F 14 280-299 55 55 55 91 91 91 F F F 15 300-319 54 54 54 87 87 87 F F F 16 320-339 55 55 55 91 91 91 F F F 17 340-359 54 54 54 89 89 89 F F F 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 54 54 54 89 89 89 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 51 51 51 83 83 83 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 55, Max flush: 91 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.035898508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:34.079272118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.079671688Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.111094793Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.111739313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.111774759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.111788887Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.111804795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} {"msg":"Crucible 6a36526a-5eef-4631-93ce-4e42906c1b06 has session id: b1b7d1d2-9417-4f21-a8ee-8ed750bede0c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112310982Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Upstairs opts: Upstairs UUID: 6a36526a-5eef-4631-93ce-4e42906c1b06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112347257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Crucible stats registered with UUID: 6a36526a-5eef-4631-93ce-4e42906c1b06","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112364335Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:34.112378853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.11281004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112835916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} {"msg":"6a36526a-5eef-4631-93ce-4e42906c1b06 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112908127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112926895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112950852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.11296594Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112980389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.112994117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113007975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113021604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113036942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113055209Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55608)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113158227Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36627)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113202171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62074)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113324216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.113981364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114022879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114054435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114082642Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114109958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114140705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114647302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114777505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.114809131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115157698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115200083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115231939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115369782Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115397548Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115412366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115425785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115439633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[0]R flush_numbers[0..12]: [91, 87, 85, 90, 80, 80, 66, 87, 89, 89, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115453471Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[0]R generation[0..12]: [55, 54, 52, 54, 48, 48, 40, 54, 54, 54, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.11546836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115482618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[1]R flush_numbers[0..12]: [91, 87, 85, 90, 80, 80, 66, 87, 89, 89, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115496486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[1]R generation[0..12]: [55, 54, 52, 54, 48, 48, 40, 54, 54, 54, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115510144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115524173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[2]R flush_numbers[0..12]: [91, 87, 85, 90, 80, 80, 66, 87, 89, 89, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115537901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[2]R generation[0..12]: [55, 54, 52, 54, 48, 48, 40, 54, 54, 54, 46, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115551539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115570197Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115584815Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115598293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115612291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.11562558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"":"downstairs","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115639118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115651547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115671584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"1","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115685113Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"client":"2","":"downstairs","session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115699521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115712079Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"6a36526a-5eef-4631-93ce-4e42906c1b06 is now active with session: b1b7d1d2-9417-4f21-a8ee-8ed750bede0c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115725068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115737146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406,"session_id":"b1b7d1d2-9417-4f21-a8ee-8ed750bede0c"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:34.115750185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 4096 data: 6 02/30 Read block 142 len 40960 03/30 Flush 04/30 Write block 80 len 12288 data: 4 4 4 05/30 Read block 456 len 40960 06/30 Write block 316 len 36864 data: 5 6 5 6 8 7 8 7 7 07/30 Read block 224 len 36864 08/30 Read block 338 len 12288 09/30 Read block 283 len 28672 10/30 Read block 220 len 4096 11/30 Read block 412 len 36864 12/30 Read block 59 len 4096 13/30 Read block 89 len 24576 14/30 Write block 432 len 36864 data: 5 5 7 8 7 6 8 7 7 15/30 Flush 16/30 Write block 500 len 24576 data: 8 7 8 9 8 7 17/30 Read block 281 len 24576 18/30 Write block 154 len 20480 data: 8 8 10 9 6 19/30 Read block 256 len 8192 20/30 Read block 533 len 4096 21/30 Write block 267 len 40960 data: 5 4 4 3 3 2 5 6 7 6 22/30 Write block 71 len 4096 data: 8 23/30 Read block 284 len 12288 24/30 Read block 31 len 36864 25/30 Read block 42 len 8192 26/30 Read block 75 len 4096 27/30 Write block 443 len 4096 data: 5 28/30 Write block 404 len 36864 data: 7 5 6 5 5 5 4 5 5 29/30 Write block 212 len 20480 data: 9 9 6 5 5 30/30 Write block 70 len 32768 data: 7 9 7 6 7 6 6 5 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:34.135116948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 F F F 1 020-039 54 54 54 87 87 87 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 54 54 54 90 90 90 T T T 4 080-099 56 56 56 93 93 93 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 54 54 54 87 87 87 T T T 8 160-179 54 54 54 89 89 89 F F F 9 180-199 54 54 54 89 89 89 F F F 10 200-219 46 46 46 78 78 78 T T T 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 53 53 53 86 86 86 T T T 14 280-299 56 56 56 92 92 92 F F F 15 300-319 56 56 56 93 93 93 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 54 54 54 89 89 89 F F F 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 54 54 54 89 89 89 T T T 21 420-439 56 56 56 93 93 93 F F F 22 440-459 56 56 56 93 93 93 T T T 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 51 51 51 83 83 83 T T T 26 520-539 54 54 54 89 89 89 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 56, Max flush: 93 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 F F F 1 020-039 54 54 54 87 87 87 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 56 56 56 93 93 93 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 54 54 54 89 89 89 F F F 9 180-199 54 54 54 89 89 89 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 52 52 52 84 84 84 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 57 57 57 94 94 94 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 56 56 56 93 93 93 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 54 54 54 89 89 89 F F F 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 57 57 57 94 94 94 F F F 21 420-439 56 56 56 93 93 93 F F F 22 440-459 57 57 57 94 94 94 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 49 49 49 81 81 81 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 57, Max flush: 94 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.412772914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.455967426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.456677837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.456718342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.456733851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.456750328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} {"msg":"Crucible 72641b5b-2b46-4b25-b126-5ed81d270b36 has session id: ed89300e-0921-483c-a323-40926597db42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.457238648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Upstairs opts: Upstairs UUID: 72641b5b-2b46-4b25-b126-5ed81d270b36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.457275253Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Crucible stats registered with UUID: 72641b5b-2b46-4b25-b126-5ed81d270b36","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.457291781Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:38.457305529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.457934221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.457961118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} {"msg":"72641b5b-2b46-4b25-b126-5ed81d270b36 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458016941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458045728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458084013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458112529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458140316Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458167022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458192679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458215636Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458239553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.45826348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_connection connected from Ok(127.0.0.1:53829)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458291307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_connection connected from Ok(127.0.0.1:51565)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.458321983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_connection connected from Ok(127.0.0.1:46188)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.4583478Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459024296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459062641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.45915228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459175027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459267685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459288413Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.45963107Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459700542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.459940582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460247624Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.46027549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460290678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460317745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460337083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460350941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460412533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460431861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[0]R flush_numbers[0..12]: [91, 87, 85, 94, 93, 80, 66, 94, 89, 89, 94, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460445929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[0]R generation[0..12]: [55, 54, 52, 57, 56, 48, 40, 57, 54, 54, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460460077Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460474526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[1]R flush_numbers[0..12]: [91, 87, 85, 94, 93, 80, 66, 94, 89, 89, 94, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460488394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[1]R generation[0..12]: [55, 54, 52, 57, 56, 48, 40, 57, 54, 54, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460502092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[2]R flush_numbers[0..12]: [91, 87, 85, 94, 93, 80, 66, 94, 89, 89, 94, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460531679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[2]R generation[0..12]: [55, 54, 52, 57, 56, 48, 40, 57, 54, 54, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460545257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460559155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460572294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460588751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.46060212Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:42:38.460615528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"":"downstairs","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460632606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460645214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"0","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460658833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"1","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460672171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"client":"2","":"downstairs","session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460685569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460698198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"72641b5b-2b46-4b25-b126-5ed81d270b36 is now active with session: ed89300e-0921-483c-a323-40926597db42","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460711386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460723635Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414,"session_id":"ed89300e-0921-483c-a323-40926597db42"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:38.460736293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 528 len 36864 data: 3 3 3 6 5 5 4 6 6 02/30 Read block 385 len 4096 03/30 Write block 173 len 36864 data: 3 5 3 7 7 8 7 7 8 04/30 Write block 245 len 36864 data: 3 3 3 4 5 5 3 3 3 05/30 Read block 63 len 16384 06/30 Read block 461 len 12288 07/30 Read block 17 len 28672 08/30 Write block 73 len 4096 data: 7 09/30 Write block 32 len 16384 data: 10 9 9 8 10/30 Read block 184 len 12288 11/30 Write block 295 len 24576 data: 4 4 3 3 6 7 12/30 Write block 539 len 28672 data: 5 4 4 5 6 5 4 13/30 Read block 324 len 8192 14/30 Read block 575 len 24576 15/30 Read block 101 len 40960 16/30 Read block 60 len 20480 17/30 Read block 96 len 36864 18/30 Read block 293 len 4096 19/30 Read block 42 len 40960 20/30 Read block 310 len 40960 21/30 Read block 99 len 8192 22/30 Read block 307 len 28672 23/30 Read block 73 len 40960 24/30 Write block 1 len 40960 data: 3 4 5 5 6 8 9 10 9 8 25/30 Write block 345 len 12288 data: 3 2 3 26/30 Read block 20 len 4096 27/30 Read block 488 len 40960 28/30 Write block 234 len 36864 data: 6 4 3 2 3 3 4 5 6 29/30 Read block 376 len 12288 30/30 Write block 84 len 32768 data: 4 6 8 6 7 7 5 5 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:38.482088769Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 T T T 1 020-039 54 54 54 87 87 87 T T T 2 040-059 52 52 52 85 85 85 F F F 3 060-079 57 57 57 94 94 94 T T T 4 080-099 56 56 56 93 93 93 T T T 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 54 54 54 89 89 89 T T T 9 180-199 54 54 54 89 89 89 T T T 10 200-219 57 57 57 94 94 94 F F F 11 220-239 52 52 52 84 84 84 T T T 12 240-259 48 48 48 80 80 80 T T T 13 260-279 57 57 57 94 94 94 F F F 14 280-299 56 56 56 92 92 92 T T T 15 300-319 56 56 56 93 93 93 T T T 16 320-339 56 56 56 93 93 93 F F F 17 340-359 54 54 54 89 89 89 T T T 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 57 57 57 94 94 94 F F F 21 420-439 56 56 56 93 93 93 F F F 22 440-459 57 57 57 94 94 94 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 54 54 54 89 89 89 T T T 27 540-559 49 49 49 81 81 81 T T T 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 57, Max flush: 94 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 95 95 95 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 58 58 58 95 95 95 F F F 4 080-099 58 58 58 95 95 95 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 58 58 58 95 95 95 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 58 58 58 95 95 95 F F F 12 240-259 58 58 58 95 95 95 F F F 13 260-279 57 57 57 94 94 94 F F F 14 280-299 58 58 58 95 95 95 F F F 15 300-319 58 58 58 95 95 95 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 58 58 58 95 95 95 F F F 18 360-379 54 54 54 89 89 89 F F F 19 380-399 52 52 52 85 85 85 F F F 20 400-419 57 57 57 94 94 94 F F F 21 420-439 56 56 56 93 93 93 F F F 22 440-459 57 57 57 94 94 94 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 58, Max flush: 95 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.758562712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:52.800166312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.800559323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.831361425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.831988137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.832031192Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.832051269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.832066707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} {"msg":"Crucible 8d3bf2a3-bd22-40f9-830c-28097de6d841 has session id: ea62db51-31d5-4148-9773-2a9845e07a4b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83252644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Upstairs opts: Upstairs UUID: 8d3bf2a3-bd22-40f9-830c-28097de6d841, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.832556567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Crucible stats registered with UUID: 8d3bf2a3-bd22-40f9-830c-28097de6d841","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.832571874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:52.832585123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833029048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833052885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} {"msg":"8d3bf2a3-bd22-40f9-830c-28097de6d841 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833115507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833133475Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833155582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83317109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833196517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833223753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833280086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833307303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83333693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833365726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47488)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833394142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57407)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833526606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52298)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.833552103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834006326Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834036412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834052441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834066559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834161197Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834202762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834635748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834666184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.834749824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835226785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835260491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835275889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835293736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835307614Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835321183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835389964Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835409722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[0]R flush_numbers[0..12]: [95, 95, 85, 95, 95, 80, 66, 94, 95, 95, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83542384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[0]R generation[0..12]: [58, 58, 52, 58, 58, 48, 40, 57, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835438098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835452567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[1]R flush_numbers[0..12]: [95, 95, 85, 95, 95, 80, 66, 94, 95, 95, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835466645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[1]R generation[0..12]: [58, 58, 52, 58, 58, 48, 40, 57, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835480483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835494621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[2]R flush_numbers[0..12]: [95, 95, 85, 95, 95, 80, 66, 94, 95, 95, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83550832Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[2]R generation[0..12]: [58, 58, 52, 58, 58, 48, 40, 57, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835524898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835539276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835552534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835565953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835579051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835592159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"":"downstairs","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835605628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:42:52.835617826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"0","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835636344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"1","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835649892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"client":"2","":"downstairs","session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.83566405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835676569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"8d3bf2a3-bd22-40f9-830c-28097de6d841 is now active with session: ea62db51-31d5-4148-9773-2a9845e07a4b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835689497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835701666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422,"session_id":"ea62db51-31d5-4148-9773-2a9845e07a4b"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:52.835714084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 8192 data: 4 8 02/30 Read block 151 len 40960 03/30 Read block 311 len 12288 04/30 Read block 369 len 4096 05/30 Write block 216 len 40960 data: 6 4 3 2 3 4 4 5 5 6 06/30 Write block 79 len 24576 data: 4 5 5 5 4 5 07/30 Read block 26 len 32768 08/30 Flush 09/30 Write block 194 len 32768 data: 7 6 5 5 6 5 5 6 10/30 Read block 514 len 4096 11/30 Write block 178 len 36864 data: 9 8 8 9 8 9 9 7 6 12/30 Read block 225 len 24576 13/30 Read block 131 len 16384 14/30 Flush 15/30 Write block 99 len 16384 data: 3 5 6 6 16/30 Write block 395 len 32768 data: 7 7 7 7 8 6 7 7 17/30 Write block 352 len 36864 data: 4 4 3 6 6 4 4 6 5 18/30 Write block 421 len 36864 data: 8 7 7 7 7 6 5 6 5 19/30 Write block 209 len 12288 data: 6 7 10 20/30 Write block 214 len 16384 data: 7 6 7 5 21/30 Read block 298 len 24576 22/30 Read block 259 len 4096 23/30 Write block 125 len 20480 data: 4 4 5 5 4 24/30 Read block 566 len 4096 25/30 Write block 132 len 16384 data: 5 5 5 5 26/30 Read block 110 len 36864 27/30 Write block 126 len 28672 data: 5 6 6 5 5 4 6 28/30 Read block 371 len 12288 29/30 Write block 450 len 20480 data: 4 4 5 5 5 30/30 Read block 86 len 32768 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 4 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 16 14 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:52.854889751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 95 95 95 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 60 60 60 96 96 96 T T T 5 100-119 48 48 48 80 80 80 T T T 6 120-139 40 40 40 66 66 66 T T T 7 140-159 57 57 57 94 94 94 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 60 60 60 97 97 97 T T T 11 220-239 60 60 60 96 96 96 F F F 12 240-259 58 58 58 95 95 95 F F F 13 260-279 57 57 57 94 94 94 F F F 14 280-299 58 58 58 95 95 95 F F F 15 300-319 58 58 58 95 95 95 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 58 58 58 95 95 95 T T T 18 360-379 54 54 54 89 89 89 T T T 19 380-399 52 52 52 85 85 85 T T T 20 400-419 57 57 57 94 94 94 T T T 21 420-439 56 56 56 93 93 93 T T T 22 440-459 57 57 57 94 94 94 T T T 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 60, Max flush: 97 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 95 95 95 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 61 61 61 98 98 98 F F F 6 120-139 61 61 61 98 98 98 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 61 61 61 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 58 58 58 95 95 95 F F F 13 260-279 57 57 57 94 94 94 F F F 14 280-299 58 58 58 95 95 95 F F F 15 300-319 58 58 58 95 95 95 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 61 61 61 98 98 98 F F F 18 360-379 61 61 61 98 98 98 F F F 19 380-399 61 61 61 98 98 98 F F F 20 400-419 61 61 61 98 98 98 F F F 21 420-439 61 61 61 98 98 98 F F F 22 440-459 61 61 61 98 98 98 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 F F F 28 560-579 44 44 44 75 75 75 F F F 29 580-599 37 37 37 59 59 59 F F F Max gen: 61, Max flush: 98 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.144061276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.18772644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.18845155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.188487735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.188502963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.188518121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} {"msg":"Crucible 6cf121ab-4a24-4b61-a7c2-7c414b679246 has session id: de7ebf5f-3a86-40bd-ada7-74330eabb2c1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.188960276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Upstairs opts: Upstairs UUID: 6cf121ab-4a24-4b61-a7c2-7c414b679246, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189003491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Crucible stats registered with UUID: 6cf121ab-4a24-4b61-a7c2-7c414b679246","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189023059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:57.189036727Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189565841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189594998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} {"msg":"6cf121ab-4a24-4b61-a7c2-7c414b679246 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189679127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189697785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189720542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.18973485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189749118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189773215Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189801052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189825699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189849346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189884331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57098)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189916008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64571)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.189946384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40099)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.1899757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.190692751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.190731886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.190758333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.190818975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191366847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191513549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191635134Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191661951Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191874954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191917519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.191948765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192168828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192196394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192212102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.1923091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.19279643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192817347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[0]R flush_numbers[0..12]: [95, 95, 85, 96, 98, 98, 98, 94, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192842224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[0]R generation[0..12]: [58, 58, 52, 60, 61, 61, 61, 57, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192856742Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.19287108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[1]R flush_numbers[0..12]: [95, 95, 85, 96, 98, 98, 98, 94, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192885139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[1]R generation[0..12]: [58, 58, 52, 60, 61, 61, 61, 57, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192898697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192912895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[2]R flush_numbers[0..12]: [95, 95, 85, 96, 98, 98, 98, 94, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192926873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[2]R generation[0..12]: [58, 58, 52, 60, 61, 61, 61, 57, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192940622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.19295818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192971788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.192985436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193014833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193030271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"":"downstairs","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193044109Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193062856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"0","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193076585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"1","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193089983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"client":"2","":"downstairs","session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193103522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.19311596Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"6cf121ab-4a24-4b61-a7c2-7c414b679246 is now active with session: de7ebf5f-3a86-40bd-ada7-74330eabb2c1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193128848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193141837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430,"session_id":"de7ebf5f-3a86-40bd-ada7-74330eabb2c1"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:42:57.193154535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 28672 data: 6 5 7 6 6 6 4 02/30 Write block 12 len 32768 data: 11 10 13 12 8 7 8 9 03/30 Flush 04/30 Write block 399 len 32768 data: 9 7 8 8 6 8 6 7 05/30 Read block 400 len 24576 06/30 Write block 261 len 28672 data: 5 6 7 7 7 5 6 07/30 Read block 27 len 36864 08/30 Write block 116 len 28672 data: 5 4 4 3 3 4 4 09/30 Read block 361 len 12288 10/30 Write block 588 len 12288 data: 3 5 6 11/30 Flush 12/30 Write block 405 len 36864 data: 7 8 6 6 6 5 6 6 8 13/30 Read block 461 len 12288 14/30 Write block 9 len 24576 data: 10 9 10 12 11 14 15/30 Write block 137 len 40960 data: 3 3 3 3 4 3 3 2 4 4 16/30 Flush 17/30 Write block 115 len 32768 data: 5 6 5 5 4 4 5 5 18/30 Flush 19/30 Write block 118 len 8192 data: 6 5 20/30 Read block 168 len 8192 21/30 Read block 388 len 12288 22/30 Write block 287 len 12288 data: 7 11 9 23/30 Write block 558 len 20480 data: 5 6 7 7 7 24/30 Read block 336 len 12288 25/30 Read block 563 len 24576 26/30 Write block 551 len 16384 data: 3 5 5 5 27/30 Read block 480 len 16384 28/30 Write block 253 len 16384 data: 4 3 3 5 29/30 Write block 253 len 8192 data: 5 4 30/30 Write block 395 len 36864 data: 8 8 8 8 10 8 9 9 7 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:42:57.212886123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 62 62 62 102 102 102 T T T 6 120-139 62 62 62 102 102 102 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 61 61 61 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 58 58 58 95 95 95 T T T 13 260-279 62 62 62 100 100 100 F F F 14 280-299 58 58 58 95 95 95 T T T 15 300-319 58 58 58 95 95 95 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 61 61 61 98 98 98 F F F 18 360-379 61 61 61 98 98 98 F F F 19 380-399 62 62 62 100 100 100 T T T 20 400-419 62 62 62 101 101 101 T T T 21 420-439 61 61 61 98 98 98 F F F 22 440-459 61 61 61 98 98 98 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 58 58 58 95 95 95 T T T 28 560-579 44 44 44 75 75 75 T T T 29 580-599 62 62 62 100 100 100 F F F Max gen: 62, Max flush: 102 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 63 63 63 103 103 103 F F F 6 120-139 62 62 62 102 102 102 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 61 61 61 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 63 63 63 103 103 103 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 63 63 63 103 103 103 F F F 15 300-319 58 58 58 95 95 95 F F F 16 320-339 56 56 56 93 93 93 F F F 17 340-359 61 61 61 98 98 98 F F F 18 360-379 61 61 61 98 98 98 F F F 19 380-399 63 63 63 103 103 103 F F F 20 400-419 63 63 63 103 103 103 F F F 21 420-439 61 61 61 98 98 98 F F F 22 440-459 61 61 61 98 98 98 F F F 23 460-479 51 51 51 83 83 83 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 F F F 27 540-559 63 63 63 103 103 103 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 63, Max flush: 103 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.461893969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:01.503639501Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.504035842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.535007713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.535582402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.535620127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.535634885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.535655943Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} {"msg":"Crucible f8b52a2d-fe4e-478a-a73c-a0e6b789f759 has session id: 8d49b0a1-03f2-4e59-a526-f72020bbe8c1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536153391Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Upstairs opts: Upstairs UUID: f8b52a2d-fe4e-478a-a73c-a0e6b789f759, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536184877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Crucible stats registered with UUID: f8b52a2d-fe4e-478a-a73c-a0e6b789f759","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536200585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:01.536214173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536615503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.53664644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} {"msg":"f8b52a2d-fe4e-478a-a73c-a0e6b789f759 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536714151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536743028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536782193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536826427Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536858663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.53688308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536910417Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536940963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536972299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.536998716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34370)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537031222Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62984)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537059298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39525)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537086705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537788208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537828843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537860879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537890575Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.53792796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.537953497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.538393633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.538451145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.538542404Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.538990928Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539028344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539044972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539130341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539152318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539166457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539178985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539192723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[0]R flush_numbers[0..12]: [101, 95, 85, 96, 98, 103, 102, 101, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539207061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[0]R generation[0..12]: [62, 58, 52, 60, 61, 63, 62, 62, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.53922115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539235408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[1]R flush_numbers[0..12]: [101, 95, 85, 96, 98, 103, 102, 101, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539249496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[1]R generation[0..12]: [62, 58, 52, 60, 61, 63, 62, 62, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539263234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539277243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[2]R flush_numbers[0..12]: [101, 95, 85, 96, 98, 103, 102, 101, 97, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539291171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[2]R generation[0..12]: [62, 58, 52, 60, 61, 63, 62, 62, 60, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539304989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539323467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539337125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539350523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539363632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.53937671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid"Wait for a query_work_queue command to finish before sending IO :1438,"":"downstairs","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539397508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539410006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"0","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539423345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"1","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539436443Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"client":"2","":"downstairs","session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539449781Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.5394623Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"f8b52a2d-fe4e-478a-a73c-a0e6b789f759 is now active with session: 8d49b0a1-03f2-4e59-a526-f72020bbe8c1","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539475428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539487686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438,"session_id":"8d49b0a1-03f2-4e59-a526-f72020bbe8c1"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:01.539500185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 16384 data: 5 4 4 4 02/30 Write block 303 len 4096 data: 7 03/30 Read block 344 len 36864 04/30 Write block 393 len 32768 data: 4 5 9 9 9 9 11 9 05/30 Read block 167 len 4096 06/30 Read block 251 len 32768 07/30 Read block 491 len 8192 08/30 Write block 188 len 4096 data: 6 09/30 Write block 310 len 16384 data: 6 6 6 8 10/30 Read block 361 len 32768 11/30 Write block 111 len 40960 data: 3 2 3 4 6 7 6 7 6 5 12/30 Read block 231 len 40960 13/30 Read block 6 len 24576 14/30 Write block 456 len 36864 data: 4 4 2 4 4 4 3 3 4 15/30 Read block 471 len 32768 16/30 Write block 372 len 40960 data: 8 6 4 4 5 6 5 4 4 4 17/30 Read block 547 len 28672 18/30 Read block 329 len 32768 19/30 Write block 233 len 36864 data: 5 7 5 4 3 4 4 5 6 20/30 Read block 335 len 12288 21/30 Write block 532 len 32768 data: 6 6 5 7 7 5 5 6 22/30 Read block 199 len 12288 23/30 Read block 341 len 36864 24/30 Write block 33 len 28672 data: 10 10 9 7 5 5 6 25/30 Read block 345 len 28672 26/30 Write block 204 len 40960 data: 5 6 5 7 6 7 8 11 10 10 27/30 Write block 193 len 40960 data: 7 8 7 6 6 7 6 6 7 7 28/30 Write block 414 len 12288 data: 7 7 7 29/30 Write block 363 len 12288 data: 6 7 7 30/30 Read block 132 len 36864 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:01.560887357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 58 58 58 95 95 95 T T T 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 63 63 63 103 103 103 T T T 6 120-139 62 62 62 102 102 102 T T T 7 140-159 62 62 62 101 101 101 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 T T T 10 200-219 61 61 61 98 98 98 T T T 11 220-239 60 60 60 96 96 96 T T T 12 240-259 63 63 63 103 103 103 T T T 13 260-279 62 62 62 100 100 100 F F F 14 280-299 63 63 63 103 103 103 F F F 15 300-319 58 58 58 95 95 95 T T T 16 320-339 56 56 56 93 93 93 T T T 17 340-359 61 61 61 98 98 98 T T T 18 360-379 61 61 61 98 98 98 T T T 19 380-399 63 63 63 103 103 103 T T T 20 400-419 63 63 63 103 103 103 T T T 21 420-439 61 61 61 98 98 98 F F F 22 440-459 61 61 61 98 98 98 T T T 23 460-479 51 51 51 83 83 83 T T T 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 58 58 58 95 95 95 T T T 27 540-559 63 63 63 103 103 103 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 63, Max flush: 103 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 64 64 64 104 104 104 F F F 2 040-059 52 52 52 85 85 85 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 64 64 64 104 104 104 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 64 64 64 104 104 104 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 104 104 104 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 63 63 63 103 103 103 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 64 64 64 104 104 104 F F F 17 340-359 64 64 64 104 104 104 F F F 18 360-379 64 64 64 104 104 104 F F F 19 380-399 64 64 64 104 104 104 F F F 20 400-419 64 64 64 104 104 104 F F F 21 420-439 61 61 61 98 98 98 F F F 22 440-459 64 64 64 104 104 104 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 53 53 53 86 86 86 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 64 64 64 104 104 104 F F F 27 540-559 63 63 63 103 103 103 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 64, Max flush: 104 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.890175065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.933327473Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.933987741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.934025616Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.934044994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.934061472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} {"msg":"Crucible 11faa166-6e16-4c67-a2e2-b8c335dbf522 has session id: 76f444ee-8ddc-4b83-8da1-cab2eef3d8db","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.934515045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Upstairs opts: Upstairs UUID: 11faa166-6e16-4c67-a2e2-b8c335dbf522, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.93456042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Crucible stats registered with UUID: 11faa166-6e16-4c67-a2e2-b8c335dbf522","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.934581757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:05.934596655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935012224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.93503948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} {"msg":"11faa166-6e16-4c67-a2e2-b8c335dbf522 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935107072Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935127309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935152196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935171674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935198721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935225487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935258193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.93528659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935316436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935346842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_connection connected from Ok(127.0.0.1:34906)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935376798Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_connection connected from Ok(127.0.0.1:50416)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935405615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_connection connected from Ok(127.0.0.1:53987)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935436541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.935971844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.936030587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.936106008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.936129105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.936523996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.936655889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937068078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937094285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937113532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937198032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937218959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937233537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937254125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937268183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.937788089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938255021Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938279128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[0]R flush_numbers[0..12]: [101, 104, 85, 96, 98, 104, 104, 101, 97, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938293496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[0]R generation[0..12]: [62, 64, 52, 60, 61, 64, 64, 62, 60, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938307484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938321712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[1]R flush_numbers[0..12]: [101, 104, 85, 96, 98, 104, 104, 101, 97, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938335561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[1]R generation[0..12]: [62, 64, 52, 60, 61, 64, 64, 62, 60, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938349169Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938363237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[2]R flush_numbers[0..12]: [101, 104, 85, 96, 98, 104, 104, 101, 97, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938381785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[2]R generation[0..12]: [62, 64, 52, 60, 61, 64, 64, 62, 60, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938396123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938410241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Max found gen is 65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.93842341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Generation requested: 66 >= found:65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938436978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938450096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938463255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"":"downstairs","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938476583Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938488652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"0","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938509279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"1","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938522717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"client":"2","":"downstairs","session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938536146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938548604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"11faa166-6e16-4c67-a2e2-b8c335dbf522 is now active with session: 76f444ee-8ddc-4b83-8da1-cab2eef3d8db","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938561623Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938573871Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446,"session_id":"76f444ee-8ddc-4b83-8da1-cab2eef3d8db"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:05.938588249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 36 len 4096 data: 8 02/30 Read block 49 len 12288 03/30 Write block 385 len 28672 data: 3 5 4 4 4 3 4 04/30 Read block 29 len 24576 05/30 Write block 21 len 12288 data: 9 11 11 06/30 Read block 246 len 4096 07/30 Write block 456 len 8192 data: 5 5 08/30 Read block 365 len 28672 09/30 Write block 48 len 32768 data: 7 5 6 6 5 7 5 5 10/30 Flush 11/30 Write block 495 len 28672 data: 7 7 7 7 8 9 8 12/30 Write block 46 len 32768 data: 6 6 8 6 7 7 6 8 13/30 Write block 445 len 28672 data: 5 6 6 3 3 5 5 14/30 Read block 280 len 24576 15/30 Read block 297 len 28672 16/30 Write block 103 len 40960 data: 6 6 6 6 4 3 3 3 4 3 17/30 Write block 176 len 28672 data: 9 8 10 9 9 10 9 18/30 Write block 115 len 8192 data: 7 8 19/30 Write block 539 len 36864 data: 7 5 5 6 7 6 5 2 2 20/30 Read block 421 len 36864 21/30 Write block 429 len 20480 data: 6 4 3 6 6 22/30 Read block 504 len 12288 23/30 Write block 345 len 8192 data: 4 3 24/30 Flush 25/30 Write block 493 len 4096 data: 4 26/30 Read block 536 len 16384 27/30 Read block 323 len 20480 28/30 Read block 93 len 8192 29/30 Write block 297 len 40960 data: 4 4 7 8 7 7 8 6 5 5 30/30 Write block 360 len 28672 data: 6 6 7 7 8 8 9 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:05.957508898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 66 66 66 106 106 106 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 66 66 66 106 106 106 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 66 66 66 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 104 104 104 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 63 63 63 103 103 103 T T T 15 300-319 64 64 64 104 104 104 T T T 16 320-339 64 64 64 104 104 104 F F F 17 340-359 66 66 66 106 106 106 F F F 18 360-379 64 64 64 104 104 104 T T T 19 380-399 66 66 66 105 105 105 F F F 20 400-419 64 64 64 104 104 104 F F F 21 420-439 66 66 66 106 106 106 F F F 22 440-459 66 66 66 106 106 106 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 66 66 66 106 106 106 T T T 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 66, Max flush: 106 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 101 101 101 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 66 66 66 106 106 106 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 61 61 61 98 98 98 F F F 5 100-119 66 66 66 106 106 106 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 66 66 66 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 104 104 104 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 67 67 67 107 107 107 F F F 15 300-319 67 67 67 107 107 107 F F F 16 320-339 64 64 64 104 104 104 F F F 17 340-359 66 66 66 106 106 106 F F F 18 360-379 67 67 67 107 107 107 F F F 19 380-399 66 66 66 105 105 105 F F F 20 400-419 64 64 64 104 104 104 F F F 21 420-439 66 66 66 106 106 106 F F F 22 440-459 66 66 66 106 106 106 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 67 67 67 107 107 107 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 67, Max flush: 107 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.195458968Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.239250126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.239885817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.239923673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.23993864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.239960108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} {"msg":"Crucible 458019a9-05c2-4f95-95cc-01ce7e28dda4 has session id: 55a1525f-a61c-42be-a0cf-3549cc56a748","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24042656Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Upstairs opts: Upstairs UUID: 458019a9-05c2-4f95-95cc-01ce7e28dda4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.240469145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Crucible stats registered with UUID: 458019a9-05c2-4f95-95cc-01ce7e28dda4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.240494182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:10.240516519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241052652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241079589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} {"msg":"458019a9-05c2-4f95-95cc-01ce7e28dda4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24115492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241174767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241199484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241225971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241253307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241281714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24131182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241340146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241373532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241400279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_connection connected from Ok(127.0.0.1:62905)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24146928Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_connection connected from Ok(127.0.0.1:60235)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241497217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_connection connected from Ok(127.0.0.1:43843)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.241589355Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242138257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242165304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242180532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.2421947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242665632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242700877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242774428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.242805154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243311101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243351916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243386752Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.2434039Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243416698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243430527Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243444025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243904258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243927665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[0]R flush_numbers[0..12]: [101, 105, 106, 96, 98, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243942183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[0]R generation[0..12]: [62, 66, 66, 60, 61, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243956331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243970559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[1]R flush_numbers[0..12]: [101, 105, 106, 96, 98, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.243984418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[1]R generation[0..12]: [62, 66, 66, 60, 61, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244002975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244026363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[2]R flush_numbers[0..12]: [101, 105, 106, 96, 98, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244041261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[2]R generation[0..12]: [62, 66, 66, 60, 61, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244054959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244072257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244085765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244099003Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244112172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24412527Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"":"downstairs","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244138599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"Wait for a query_work_queue command to finish before sending IO msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244168385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"0","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244183303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"1","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244196662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"client":"2","":"downstairs","session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.24421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244222448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"458019a9-05c2-4f95-95cc-01ce7e28dda4 is now active with session: 55a1525f-a61c-42be-a0cf-3549cc56a748","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244235297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244247115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454,"session_id":"55a1525f-a61c-42be-a0cf-3549cc56a748"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:10.244259244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 28672 data: 8 8 8 7 6 7 7 02/30 Read block 508 len 36864 03/30 Read block 407 len 28672 04/30 Read block 455 len 40960 05/30 Write block 316 len 24576 data: 6 7 6 7 9 8 06/30 Flush 07/30 Write block 5 len 24576 data: 7 9 10 11 11 10 08/30 Read block 356 len 20480 09/30 Read block 366 len 36864 10/30 Read block 188 len 36864 11/30 Write block 493 len 16384 data: 5 4 8 8 12/30 Read block 130 len 40960 13/30 Write block 343 len 36864 data: 4 4 5 4 4 4 4 4 5 14/30 Read block 242 len 24576 15/30 Read block 556 len 20480 16/30 Read block 443 len 24576 17/30 Read block 301 len 4096 18/30 Write block 82 len 20480 data: 6 5 6 7 9 19/30 Write block 390 len 36864 data: 4 5 4 5 6 10 10 10 10 20/30 Write block 447 len 4096 data: 7 21/30 Read block 229 len 4096 22/30 Flush 23/30 Write block 447 len 16384 data: 8 4 4 6 24/30 Read block 272 len 28672 25/30 Flush 26/30 Write block 376 len 40960 data: 6 7 6 5 5 5 5 5 3 4 27/30 Write block 413 len 4096 data: 9 28/30 Read block 383 len 28672 29/30 Read block 355 len 28672 30/30 Read block 168 len 40960 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:10.265222589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 109 109 109 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 66 66 66 106 106 106 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 68 68 68 109 109 109 F F F 5 100-119 66 66 66 106 106 106 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 66 66 66 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 104 104 104 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 67 67 67 107 107 107 F F F 15 300-319 68 68 68 108 108 108 F F F 16 320-339 68 68 68 108 108 108 F F F 17 340-359 68 68 68 109 109 109 F F F 18 360-379 67 67 67 107 107 107 T T T 19 380-399 68 68 68 109 109 109 T T T 20 400-419 64 64 64 104 104 104 T T T 21 420-439 68 68 68 108 108 108 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 68, Max flush: 110 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 109 109 109 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 66 66 66 106 106 106 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 68 68 68 109 109 109 F F F 5 100-119 66 66 66 106 106 106 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 101 101 101 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 66 66 66 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 104 104 104 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 67 67 67 107 107 107 F F F 15 300-319 68 68 68 108 108 108 F F F 16 320-339 68 68 68 108 108 108 F F F 17 340-359 68 68 68 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 69 69 69 111 111 111 F F F 21 420-439 68 68 68 108 108 108 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 62 62 62 100 100 100 F F F Max gen: 69, Max flush: 111 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.491967311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.535481073Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.536403819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.536441944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.536456862Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.53647227Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} {"msg":"Crucible 711838a6-253b-4335-99bf-1d42916e6420 has session id: 25d21a33-7f84-4d69-b71e-96b10f743799","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537007474Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Upstairs opts: Upstairs UUID: 711838a6-253b-4335-99bf-1d42916e6420, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537049329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Crucible stats registered with UUID: 711838a6-253b-4335-99bf-1d42916e6420","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537078175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:14.537100972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537581112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537611729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} {"msg":"711838a6-253b-4335-99bf-1d42916e6420 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537677531Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537697578Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537720565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537736243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537751791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537772359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537798685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537826272Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537851999Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537879435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_connection connected from Ok(127.0.0.1:63119)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.53792687Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_connection connected from Ok(127.0.0.1:65346)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.537969074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_connection connected from Ok(127.0.0.1:59228)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.538056584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.538866023Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.538903748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.538930785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.538949382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.539626838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.539658305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.539673863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.53969403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540241332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540269838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540284467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540356988Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540377855Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540391953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540410261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540878513Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.54090297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[0]R flush_numbers[0..12]: [109, 105, 106, 96, 109, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540917558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[0]R generation[0..12]: [68, 66, 66, 60, 68, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540931876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540945994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[1]R flush_numbers[0..12]: [109, 105, 106, 96, 109, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540959783Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[1]R generation[0..12]: [68, 66, 66, 60, 68, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540973711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.540988099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[2]R flush_numbers[0..12]: [109, 105, 106, 96, 109, 106, 104, 101, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541002407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[2]R generation[0..12]: [68, 66, 66, 60, 68, 66, 64, 62, 66, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541016156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541030144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541043352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541057281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541075788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541089876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"":"downstairs","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541103365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541115903Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"0","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541130231Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"1","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-12-11T23:43:14.541152059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"client":"2","":"downstairs","session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541167257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541180675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"711838a6-253b-4335-99bf-1d42916e6420 is now active with session: 25d21a33-7f84-4d69-b71e-96b10f743799","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541193884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541206272Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462,"session_id":"25d21a33-7f84-4d69-b71e-96b10f743799"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:14.541219171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 40960 data: 4 5 6 8 9 8 7 7 8 7 02/30 Read block 378 len 28672 03/30 Read block 313 len 32768 04/30 Read block 206 len 12288 05/30 Flush 06/30 Write block 442 len 20480 data: 6 6 6 6 7 07/30 Read block 18 len 28672 08/30 Read block 420 len 36864 09/30 Read block 217 len 40960 10/30 Read block 565 len 24576 11/30 Read block 423 len 4096 12/30 Write block 226 len 16384 data: 7 6 6 7 13/30 Read block 383 len 20480 14/30 Write block 225 len 28672 data: 7 8 7 7 8 7 8 15/30 Read block 518 len 36864 16/30 Write block 581 len 16384 data: 3 3 2 2 17/30 Read block 366 len 8192 18/30 Read block 393 len 40960 19/30 Write block 136 len 20480 data: 5 4 4 4 4 20/30 Read block 94 len 4096 21/30 Flush 22/30 Write block 61 len 16384 data: 9 9 6 5 23/30 Read block 384 len 28672 24/30 Write block 101 len 24576 data: 7 7 7 7 7 7 25/30 Write block 42 len 8192 data: 4 6 26/30 Flush 27/30 Write block 3 len 36864 data: 6 6 8 10 11 12 12 11 11 28/30 Write block 399 len 16384 data: 12 10 10 10 29/30 Write block 82 len 20480 data: 7 6 7 8 10 30/30 Read block 208 len 12288 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:14.562299351Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 109 109 109 T T T 1 020-039 66 66 66 105 105 105 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 70 70 70 114 114 114 F F F 4 080-099 68 68 68 109 109 109 T T T 5 100-119 70 70 70 114 114 114 F F F 6 120-139 70 70 70 113 113 113 F F F 7 140-159 70 70 70 113 113 113 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 70 70 70 113 113 113 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 67 67 67 107 107 107 F F F 15 300-319 68 68 68 108 108 108 F F F 16 320-339 68 68 68 108 108 108 F F F 17 340-359 68 68 68 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 T T T 20 400-419 69 69 69 111 111 111 T T T 21 420-439 68 68 68 108 108 108 F F F 22 440-459 70 70 70 113 113 113 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 70 70 70 113 113 113 F F F Max gen: 70, Max flush: 114 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 70 70 70 114 114 114 F F F 4 080-099 71 71 71 115 115 115 F F F 5 100-119 70 70 70 114 114 114 F F F 6 120-139 70 70 70 113 113 113 F F F 7 140-159 70 70 70 113 113 113 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 70 70 70 113 113 113 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 67 67 67 107 107 107 F F F 15 300-319 68 68 68 108 108 108 F F F 16 320-339 68 68 68 108 108 108 F F F 17 340-359 68 68 68 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 71 71 71 115 115 115 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 68 68 68 108 108 108 F F F 22 440-459 70 70 70 113 113 113 F F F 23 460-479 64 64 64 104 104 104 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 63 63 63 103 103 103 F F F 29 580-599 70 70 70 113 113 113 F F F Max gen: 71, Max flush: 115 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.79718839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.841592383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842262719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842300704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842315593Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842331521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} {"msg":"Crucible ffa0f698-c65e-4814-bfe0-e773a1c34289 has session id: fcc78494-84b5-4546-b384-b71bb0a4b947","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.8428166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Upstairs opts: Upstairs UUID: ffa0f698-c65e-4814-bfe0-e773a1c34289, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842859935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Crucible stats registered with UUID: ffa0f698-c65e-4814-bfe0-e773a1c34289","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.842878713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:18.842894621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843360713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843403587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} {"msg":"ffa0f698-c65e-4814-bfe0-e773a1c34289 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843473659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843501595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843526172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.84354228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843557668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843575516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843601383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843629399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843658736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843686232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_connection connected from Ok(127.0.0.1:48026)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843732366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_connection connected from Ok(127.0.0.1:56064)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843839673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_connection connected from Ok(127.0.0.1:50724)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.843915444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.844408243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.844445078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.844523398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.844562893Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.84459129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.844620266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845096997Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845166238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845190285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845690813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845729818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845759765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845785311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845808619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845832006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845854333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.84587934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[0]R flush_numbers[0..12]: [115, 105, 114, 114, 115, 114, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845907716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[0]R generation[0..12]: [71, 66, 70, 70, 71, 70, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845931603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.84595731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[1]R flush_numbers[0..12]: [115, 105, 114, 114, 115, 114, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.845984337Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[1]R generation[0..12]: [71, 66, 70, 70, 71, 70, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846031231Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846064237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R flush_numbers[0..12]: [115, 105, 114, 114, 115, 114, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846100182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[2]R generation[0..12]: [71, 66, 70, 70, 71, 70, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846130728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846161185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846189221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846216778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846243805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"All extents match"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846274921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"":"downstairs","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846301437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846318665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"0","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846332743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"1","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846346262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"client":"2","":"downstairs","session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.84636001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846372818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"ffa0f698-c65e-4814-bfe0-e773a1c34289 is now active with session: fcc78494-84b5-4546-b384-b71bb0a4b947","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846385597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846397675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470,"session_id":"fcc78494-84b5-4546-b384-b71bb0a4b947"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:18.846410204Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 20480 data: 8 8 5 7 7 02/30 Write block 97 len 32768 data: 4 4 4 6 8 8 8 8 03/30 Read block 97 len 40960 04/30 Write block 35 len 4096 data: 10 05/30 Read block 47 len 36864 06/30 Write block 571 len 12288 data: 6 6 6 07/30 Read block 313 len 20480 08/30 Read block 195 len 40960 09/30 Read block 350 len 4096 10/30 Read block 274 len 36864 11/30 Write block 370 len 40960 data: 9 9 9 7 5 5 7 8 7 6 12/30 Read block 220 len 40960 13/30 Write block 294 len 36864 data: 4 5 5 5 5 8 9 8 8 14/30 Read block 288 len 32768 15/30 Read block 347 len 40960 16/30 Write block 348 len 4096 data: 5 17/30 Read block 405 len 4096 18/30 Read block 362 len 4096 19/30 Write block 456 len 28672 data: 6 6 3 5 5 5 4 20/30 Write block 64 len 8192 data: 6 3 21/30 Flush 22/30 Write block 460 len 4096 data: 6 23/30 Read block 92 len 16384 24/30 Read block 295 len 36864 25/30 Write block 111 len 16384 data: 5 4 4 5 26/30 Read block 51 len 16384 27/30 Write block 594 len 20480 data: 8 6 4 4 3 28/30 Read block 62 len 16384 29/30 Write block 473 len 16384 data: 6 5 5 5 30/30 Write block 327 len 12288 data: 8 6 5 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:18.86535461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 72 72 72 116 116 116 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 72 72 72 116 116 116 T T T 6 120-139 70 70 70 113 113 113 F F F 7 140-159 70 70 70 113 113 113 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 70 70 70 113 113 113 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 72 72 72 116 116 116 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 68 68 68 108 108 108 T T T 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 71 71 71 115 115 115 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 72 72 72 116 116 116 F F F 23 460-479 72 72 72 116 116 116 T T T 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 72 72 72 116 116 116 F F F 29 580-599 70 70 70 113 113 113 T T T Max gen: 72, Max flush: 116 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 72 72 72 116 116 116 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 73 73 73 117 117 117 F F F 6 120-139 70 70 70 113 113 113 F F F 7 140-159 70 70 70 113 113 113 F F F 8 160-179 66 66 66 106 106 106 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 70 70 70 113 113 113 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 72 72 72 116 116 116 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 73 73 73 117 117 117 F F F 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 71 71 71 115 115 115 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 72 72 72 116 116 116 F F F 23 460-479 73 73 73 117 117 117 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 66 66 66 106 106 106 F F F 27 540-559 66 66 66 106 106 106 F F F 28 560-579 72 72 72 116 116 116 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 73, Max flush: 117 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.098095767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:23.140252868Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.140623292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.171683352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17226752Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172304075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172318613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172333782Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} {"msg":"Crucible f4d5f559-00fd-426d-8d53-061f50c747a2 has session id: 35f8a5ad-2265-451c-94c1-3a94312ca9da","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172779116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Upstairs opts: Upstairs UUID: f4d5f559-00fd-426d-8d53-061f50c747a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172814682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Crucible stats registered with UUID: f4d5f559-00fd-426d-8d53-061f50c747a2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.172841798Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:23.172865395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173254547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173284153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} {"msg":"f4d5f559-00fd-426d-8d53-061f50c747a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173340276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173358804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17338897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173416987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173444064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17347027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173497717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173523284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173545021Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173609503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_connection connected from Ok(127.0.0.1:33826)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173643469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_connection connected from Ok(127.0.0.1:46905)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173674225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_connection connected from Ok(127.0.0.1:55618)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.173704791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174394175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174422232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174441369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174455608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174476915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174503412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.174994341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175151801Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175260528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17548393Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175524755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175543163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175671177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175698493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175712791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175737079Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175751607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[0]R flush_numbers[0..12]: [115, 116, 114, 116, 116, 117, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175791242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[0]R generation[0..12]: [71, 72, 70, 72, 72, 73, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175813029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175827847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[1]R flush_numbers[0..12]: [115, 116, 114, 116, 116, 117, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175841895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[1]R generation[0..12]: [71, 72, 70, 72, 72, 73, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175856884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175871092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[2]R flush_numbers[0..12]: [115, 116, 114, 116, 116, 117, 113, 113, 106, 112, 104, 113]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17588482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[2]R generation[0..12]: [71, 72, 70, 72, 72, 73, 70, 70, 66, 70, 64, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175898739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175912817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175925885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175939363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175952692Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17596606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"":"downstairs","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.175979449Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176022403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"0","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.17604379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"1","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176057819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"client":"2","":"downstairs","session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176071187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176083725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"f4d5f559-00fd-426d-8d53-061f50c747a2 is now active with session: 35f8a5ad-2265-451c-94c1-3a94312ca9da","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176097824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176111122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478,"session_id":"35f8a5ad-2265-451c-94c1-3a94312ca9da"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:23.176123611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 20480 data: 8 9 9 11 10 02/30 Read block 304 len 12288 03/30 Write block 547 len 4096 data: 3 04/30 Read block 441 len 16384 05/30 Read block 76 len 32768 06/30 Write block 443 len 12288 data: 8 7 7 07/30 Write block 516 len 4096 data: 6 08/30 Write block 148 len 24576 data: 6 8 6 7 8 9 09/30 Read block 10 len 20480 10/30 Write block 384 len 40960 data: 4 5 6 5 5 5 5 6 5 6 11/30 Write block 571 len 36864 data: 7 7 7 5 4 6 7 5 4 12/30 Read block 595 len 16384 13/30 Write block 172 len 12288 data: 3 4 6 14/30 Write block 295 len 20480 data: 6 6 6 6 9 15/30 Read block 209 len 16384 16/30 Read block 536 len 40960 17/30 Write block 519 len 12288 data: 6 5 6 18/30 Write block 226 len 4096 data: 9 19/30 Write block 119 len 32768 data: 7 6 6 6 4 4 5 6 20/30 Read block 588 len 8192 21/30 Read block 448 len 36864 22/30 Read block 453 len 20480 23/30 Write block 545 len 28672 data: 6 3 4 3 4 3 4 24/30 Read block 278 len 16384 25/30 Read block 540 len 8192 26/30 Read block 48 len 20480 27/30 Read block 508 len 8192 28/30 Write block 456 len 28672 data: 7 7 4 6 7 6 5 29/30 Write block 24 len 16384 data: 10 11 9 10 30/30 Read block 464 len 20480 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:23.1952764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 72 72 72 116 116 116 T T T 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 73 73 73 117 117 117 T T T 6 120-139 70 70 70 113 113 113 T T T 7 140-159 70 70 70 113 113 113 T T T 8 160-179 66 66 66 106 106 106 T T T 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 70 70 70 113 113 113 T T T 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 72 72 72 116 116 116 T T T 15 300-319 72 72 72 116 116 116 F F F 16 320-339 73 73 73 117 117 117 F F F 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 71 71 71 115 115 115 T T T 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 72 72 72 116 116 116 T T T 23 460-479 73 73 73 117 117 117 T T T 24 480-499 68 68 68 109 109 109 F F F 25 500-519 66 66 66 106 106 106 T T T 26 520-539 66 66 66 106 106 106 T T T 27 540-559 66 66 66 106 106 106 T T T 28 560-579 72 72 72 116 116 116 T T T 29 580-599 73 73 73 117 117 117 F F F Max gen: 73, Max flush: 117 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 74 74 74 118 118 118 F F F 8 160-179 74 74 74 118 118 118 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 74 74 74 118 118 118 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 74 74 74 118 118 118 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 73 73 73 117 117 117 F F F 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 74 74 74 118 118 118 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 74 74 74 118 118 118 F F F 23 460-479 74 74 74 118 118 118 F F F 24 480-499 68 68 68 109 109 109 F F F 25 500-519 74 74 74 118 118 118 F F F 26 520-539 74 74 74 118 118 118 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 74 74 74 118 118 118 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 74, Max flush: 118 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.510822558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.554781145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.555623891Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.555664606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.555679544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.555695002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} {"msg":"Crucible 8c19a4fa-2e4c-4b27-a5ad-22d68c9ead39 has session id: 4e2a984f-3710-4402-8910-17f53b1e3905","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556150695Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Upstairs opts: Upstairs UUID: 8c19a4fa-2e4c-4b27-a5ad-22d68c9ead39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.55619384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Crucible stats registered with UUID: 8c19a4fa-2e4c-4b27-a5ad-22d68c9ead39","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556216987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:27.556235155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556738912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556767289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} {"msg":"8c19a4fa-2e4c-4b27-a5ad-22d68c9ead39 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556821532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556848198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556878525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556905921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.556933278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557030296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557058722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557084799Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557110566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557134483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_connection connected from Ok(127.0.0.1:56248)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.55715918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_connection connected from Ok(127.0.0.1:47991)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557188016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:27.557260917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:27.557308571Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:27.557337947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557511286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557929364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557957121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557973279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.557987357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.558661063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.55868637Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.55924621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.559275247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.559290285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.559303273Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.559316882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:27.559334299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.558372339Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_connection connected from Ok(127.0.0.1:34422)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.558452879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.55965577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.559693815Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560230948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560810756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560833173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[0]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 118, 118, 112, 104, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.56085888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[0]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 74, 74, 70, 64, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560874558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560889496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[1]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 118, 118, 112, 104, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560903824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[1]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 74, 74, 70, 64, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560918033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560932461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[2]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 118, 118, 112, 104, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560946639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[2]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 74, 74, 70, 64, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560966167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560981345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Max found gen is 75","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.560996783Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Generation requested: 76 >= found:75","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.56102213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561038618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561052506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"":"downstairs","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561066434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561078933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"0","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561092631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"1","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561116248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"client":"2","":"downstairs","session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:43:37.561144525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561172081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"8c19a4fa-2e4c-4b27-a5ad-22d68c9ead39 is now active with session: 4e2a984f-3710-4402-8910-17f53b1e3905","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561194168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561211876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486,"session_id":"4e2a984f-3710-4402-8910-17f53b1e3905"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:37.561235243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 24576 data: 7 7 7 8 7 8 02/30 Write block 468 len 20480 data: 6 6 5 5 7 03/30 Write block 292 len 20480 data: 4 4 5 7 7 04/30 Write block 291 len 16384 data: 6 5 5 6 05/30 Read block 250 len 16384 06/30 Write block 484 len 36864 data: 6 6 5 4 4 4 4 3 3 07/30 Flush 08/30 Write block 447 len 36864 data: 9 5 5 7 6 6 6 6 4 09/30 Read block 494 len 32768 10/30 Write block 399 len 4096 data: 13 11/30 Read block 501 len 32768 12/30 Write block 521 len 28672 data: 7 5 5 4 3 4 5 13/30 Write block 169 len 4096 data: 5 14/30 Write block 235 len 8192 data: 6 5 15/30 Read block 32 len 32768 16/30 Read block 579 len 12288 17/30 Read block 239 len 12288 18/30 Write block 325 len 20480 data: 7 8 9 7 6 19/30 Write block 152 len 16384 data: 9 10 10 10 20/30 Write block 518 len 32768 data: 7 7 6 8 6 6 5 4 21/30 Write block 472 len 36864 data: 8 7 6 6 6 7 8 8 8 22/30 Write block 486 len 36864 data: 6 5 5 5 5 4 4 6 5 23/30 Write block 281 len 20480 data: 5 7 6 7 7 24/30 Read block 99 len 36864 25/30 Read block 75 len 12288 26/30 Read block 306 len 4096 27/30 Read block 103 len 20480 28/30 Read block 343 len 24576 29/30 Read block 587 len 12288 30/30 Write block 144 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 9 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:37.580226783Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 74 74 74 118 118 118 T T T 8 160-179 74 74 74 118 118 118 T T T 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 74 74 74 118 118 118 T T T 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 76 76 76 119 119 119 T T T 15 300-319 72 72 72 116 116 116 F F F 16 320-339 73 73 73 117 117 117 T T T 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 74 74 74 118 118 118 T T T 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 74 74 74 118 118 118 T T T 23 460-479 76 76 76 119 119 119 T T T 24 480-499 76 76 76 119 119 119 T T T 25 500-519 74 74 74 118 118 118 T T T 26 520-539 74 74 74 118 118 118 T T T 27 540-559 74 74 74 118 118 118 F F F 28 560-579 74 74 74 118 118 118 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 76, Max flush: 119 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 70 70 70 114 114 114 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 77 77 77 120 120 120 F F F 8 160-179 77 77 77 120 120 120 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 77 77 77 120 120 120 F F F 12 240-259 64 64 64 104 104 104 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 77 77 77 120 120 120 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 77 77 77 120 120 120 F F F 17 340-359 72 72 72 116 116 116 F F F 18 360-379 72 72 72 116 116 116 F F F 19 380-399 77 77 77 120 120 120 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 77 77 77 120 120 120 F F F 23 460-479 77 77 77 120 120 120 F F F 24 480-499 77 77 77 120 120 120 F F F 25 500-519 77 77 77 120 120 120 F F F 26 520-539 77 77 77 120 120 120 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 74 74 74 118 118 118 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 77, Max flush: 120 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.853090434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.898527398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899324519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899362294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899382541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899398929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} {"msg":"Crucible 2bef8d84-cfb0-4958-b72f-b90f6db12335 has session id: 845c7819-55dd-4b55-a582-fdcd38a17b5f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899803619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Upstairs opts: Upstairs UUID: 2bef8d84-cfb0-4958-b72f-b90f6db12335, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899846154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Crucible stats registered with UUID: 2bef8d84-cfb0-4958-b72f-b90f6db12335","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.899870171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:41.899891048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900408064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900445279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} {"msg":"2bef8d84-cfb0-4958-b72f-b90f6db12335 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900500052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900531838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900571193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.90060062Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900629806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900657693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900692588Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900722595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900749111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900774738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37302)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900802045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61331)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900830901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60075)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.900861068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.901501218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.901530094Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.901545812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.901560121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902236797Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902272112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902663004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902699829Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902892165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902925201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902952308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.902977705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903006991Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903036247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903275308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903760847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903781274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[0]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 120, 120, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903798483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[0]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 77, 77, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903813031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903827419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[1]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 120, 120, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903841667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[1]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 77, 77, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903855325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903869334Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[2]R flush_numbers[0..12]: [115, 118, 114, 116, 116, 118, 118, 120, 120, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903886712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[2]R generation[0..12]: [71, 74, 70, 72, 72, 74, 74, 77, 77, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.90390076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903915058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903928486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903941855Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903955063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.903968041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"":"downstairs","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.90398127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-11T23:43:41.904053841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"0","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904076738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"1","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904090716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"client":"2","":"downstairs","session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904104094Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904116713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"2bef8d84-cfb0-4958-b72f-b90f6db12335 is now active with session: 845c7819-55dd-4b55-a582-fdcd38a17b5f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904130101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904155428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494,"session_id":"845c7819-55dd-4b55-a582-fdcd38a17b5f"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:41.904171806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 32768 data: 7 7 5 4 4 4 4 5 02/30 Read block 271 len 32768 03/30 Write block 455 len 32768 data: 5 8 8 5 7 8 7 6 04/30 Read block 342 len 28672 05/30 Read block 385 len 20480 06/30 Read block 375 len 8192 07/30 Write block 42 len 16384 data: 5 7 6 7 08/30 Read block 495 len 12288 09/30 Read block 406 len 28672 10/30 Read block 354 len 12288 11/30 Write block 487 len 40960 data: 6 6 6 6 5 5 7 6 9 9 12/30 Read block 583 len 40960 13/30 Flush 14/30 Write block 43 len 40960 data: 8 7 8 7 7 9 7 8 8 7 15/30 Write block 105 len 40960 data: 8 8 5 4 4 4 6 5 5 6 16/30 Read block 21 len 32768 17/30 Write block 340 len 4096 data: 5 18/30 Write block 161 len 28672 data: 5 4 4 7 7 6 6 19/30 Read block 164 len 8192 20/30 Read block 566 len 20480 21/30 Read block 556 len 16384 22/30 Read block 124 len 12288 23/30 Flush 24/30 Write block 53 len 24576 data: 9 6 6 5 6 6 25/30 Write block 351 len 40960 data: 6 5 5 4 7 7 5 5 7 7 26/30 Read block 467 len 20480 27/30 Write block 559 len 24576 data: 7 8 8 8 6 8 28/30 Write block 249 len 8192 data: 6 6 29/30 Read block 560 len 32768 30/30 Read block 458 len 20480 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:41.925925763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 78 78 78 122 122 122 T T T 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 78 78 78 122 122 122 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 77 77 77 120 120 120 F F F 8 160-179 78 78 78 122 122 122 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 77 77 77 120 120 120 F F F 12 240-259 78 78 78 121 121 121 T T T 13 260-279 62 62 62 100 100 100 F F F 14 280-299 77 77 77 120 120 120 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 77 77 77 120 120 120 F F F 17 340-359 78 78 78 122 122 122 T T T 18 360-379 72 72 72 116 116 116 T T T 19 380-399 77 77 77 120 120 120 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 78 78 78 121 121 121 F F F 23 460-479 78 78 78 121 121 121 F F F 24 480-499 78 78 78 121 121 121 F F F 25 500-519 77 77 77 120 120 120 F F F 26 520-539 77 77 77 120 120 120 F F F 27 540-559 74 74 74 118 118 118 T T T 28 560-579 74 74 74 118 118 118 T T T 29 580-599 73 73 73 117 117 117 F F F Max gen: 78, Max flush: 122 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 115 115 115 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 79 79 79 123 123 123 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 78 78 78 122 122 122 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 77 77 77 120 120 120 F F F 8 160-179 78 78 78 122 122 122 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 77 77 77 120 120 120 F F F 12 240-259 79 79 79 123 123 123 F F F 13 260-279 62 62 62 100 100 100 F F F 14 280-299 77 77 77 120 120 120 F F F 15 300-319 72 72 72 116 116 116 F F F 16 320-339 77 77 77 120 120 120 F F F 17 340-359 79 79 79 123 123 123 F F F 18 360-379 79 79 79 123 123 123 F F F 19 380-399 77 77 77 120 120 120 F F F 20 400-419 71 71 71 115 115 115 F F F 21 420-439 72 72 72 116 116 116 F F F 22 440-459 78 78 78 121 121 121 F F F 23 460-479 78 78 78 121 121 121 F F F 24 480-499 78 78 78 121 121 121 F F F 25 500-519 77 77 77 120 120 120 F F F 26 520-539 77 77 77 120 120 120 F F F 27 540-559 79 79 79 123 123 123 F F F 28 560-579 79 79 79 123 123 123 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 79, Max flush: 123 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.162858719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.206873049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.207572202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.207608928Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.207623546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.207644864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} {"msg":"Crucible 14333c43-3793-4aba-9ecd-ed7d2df19c8d has session id: 0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208116395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Upstairs opts: Upstairs UUID: 14333c43-3793-4aba-9ecd-ed7d2df19c8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.20815536Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Crucible stats registered with UUID: 14333c43-3793-4aba-9ecd-ed7d2df19c8d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208174938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:46.208189046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208767154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208804169Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} {"msg":"14333c43-3793-4aba-9ecd-ed7d2df19c8d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208861862Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208889969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208914236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208929414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208943772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.20895775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208971749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.208985187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209003615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209017983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33112)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209054078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52243)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209275521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59467)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209308627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.20976517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209806255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209828342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.20984445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209872837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.209888505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.210373325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.210432137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.210542333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211032883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211064139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211079467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211092205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211105794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211119392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.21113173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211144998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[0]R flush_numbers[0..12]: [115, 118, 123, 116, 116, 122, 118, 120, 122, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211159147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[0]R generation[0..12]: [71, 74, 79, 72, 72, 78, 74, 77, 78, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211173195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211187493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[1]R flush_numbers[0..12]: [115, 118, 123, 116, 116, 122, 118, 120, 122, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211202841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[1]R generation[0..12]: [71, 74, 79, 72, 72, 78, 74, 77, 78, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.21121674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211230958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[2]R flush_numbers[0..12]: [115, 118, 123, 116, 116, 122, 118, 120, 122, 112, 104, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211244686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[2]R generation[0..12]: [71, 74, 79, 72, 72, 78, 74, 77, 78, 70, 64, 77]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211258475Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211275062Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211288441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211301959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211315108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:43:46.211328386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"":"downstairs","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211344944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211357532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"0","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.21137141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"1","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211384869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"client":"2","":"downstairs","session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211398407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211410825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"14333c43-3793-4aba-9ecd-ed7d2df19c8d is now active with session: 0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211423914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211436173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502,"session_id":"0dc72dbb-e3ff-4e7c-bba5-8f7363125f4e"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:46.211448481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 20480 data: 5 5 6 5 5 02/30 Read block 147 len 16384 03/30 Write block 318 len 24576 data: 7 8 10 9 9 8 04/30 Read block 534 len 20480 05/30 Write block 17 len 28672 data: 8 9 10 10 10 12 12 06/30 Flush 07/30 Write block 250 len 36864 data: 7 4 4 6 5 4 6 5 4 08/30 Read block 215 len 4096 09/30 Write block 414 len 8192 data: 8 8 10/30 Flush 11/30 Flush 12/30 Write block 532 len 20480 data: 7 7 6 8 8 13/30 Read block 181 len 20480 14/30 Read block 45 len 4096 15/30 Read block 464 len 40960 16/30 Read block 19 len 24576 17/30 Write block 435 len 40960 data: 9 8 7 9 9 9 6 8 9 8 18/30 Write block 227 len 16384 data: 8 8 9 8 19/30 Read block 178 len 16384 20/30 Write block 452 len 28672 data: 7 7 7 6 9 9 6 21/30 Read block 405 len 28672 22/30 Write block 352 len 20480 data: 6 6 5 8 8 23/30 Read block 587 len 16384 24/30 Read block 437 len 8192 25/30 Read block 591 len 32768 26/30 Write block 148 len 12288 data: 7 9 7 27/30 Read block 363 len 36864 28/30 Write block 282 len 16384 data: 8 7 8 8 29/30 Read block 205 len 4096 30/30 Write block 264 len 40960 data: 8 8 6 7 5 5 4 4 3 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:46.231470553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 80 80 80 124 124 124 F F F 2 040-059 79 79 79 123 123 123 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 78 78 78 122 122 122 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 77 77 77 120 120 120 T T T 8 160-179 78 78 78 122 122 122 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 77 77 77 120 120 120 T T T 12 240-259 80 80 80 125 125 125 F F F 13 260-279 62 62 62 100 100 100 T T T 14 280-299 77 77 77 120 120 120 T T T 15 300-319 80 80 80 124 124 124 F F F 16 320-339 80 80 80 124 124 124 F F F 17 340-359 80 80 80 124 124 124 T T T 18 360-379 79 79 79 123 123 123 F F F 19 380-399 77 77 77 120 120 120 F F F 20 400-419 80 80 80 125 125 125 F F F 21 420-439 72 72 72 116 116 116 T T T 22 440-459 78 78 78 121 121 121 T T T 23 460-479 78 78 78 121 121 121 F F F 24 480-499 78 78 78 121 121 121 F F F 25 500-519 77 77 77 120 120 120 F F F 26 520-539 77 77 77 120 120 120 T T T 27 540-559 79 79 79 123 123 123 F F F 28 560-579 79 79 79 123 123 123 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 80, Max flush: 125 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 80 80 80 124 124 124 F F F 2 040-059 79 79 79 123 123 123 F F F 3 060-079 72 72 72 116 116 116 F F F 4 080-099 72 72 72 116 116 116 F F F 5 100-119 78 78 78 122 122 122 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 81 81 81 126 126 126 F F F 8 160-179 78 78 78 122 122 122 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 81 81 81 126 126 126 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 81 81 81 126 126 126 F F F 14 280-299 81 81 81 126 126 126 F F F 15 300-319 80 80 80 124 124 124 F F F 16 320-339 80 80 80 124 124 124 F F F 17 340-359 81 81 81 126 126 126 F F F 18 360-379 79 79 79 123 123 123 F F F 19 380-399 77 77 77 120 120 120 F F F 20 400-419 80 80 80 125 125 125 F F F 21 420-439 81 81 81 126 126 126 F F F 22 440-459 81 81 81 126 126 126 F F F 23 460-479 78 78 78 121 121 121 F F F 24 480-499 78 78 78 121 121 121 F F F 25 500-519 77 77 77 120 120 120 F F F 26 520-539 81 81 81 126 126 126 F F F 27 540-559 79 79 79 123 123 123 F F F 28 560-579 79 79 79 123 123 123 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 81, Max flush: 126 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.486161142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.529825456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.530674061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.530713296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.530728644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.530743942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} {"msg":"Crucible c2dc649d-a790-4657-8199-5702242a6c22 has session id: 9eb644b3-7e7d-4b38-8d95-731446196393","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531190487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Upstairs opts: Upstairs UUID: c2dc649d-a790-4657-8199-5702242a6c22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531234741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Crucible stats registered with UUID: c2dc649d-a790-4657-8199-5702242a6c22","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531261418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:50.531281035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531791502Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531816509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} {"msg":"c2dc649d-a790-4657-8199-5702242a6c22 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531895049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531924166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531948483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531963281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531977529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.531991067Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532004805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532025613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532058469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532079166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_connection connected from Ok(127.0.0.1:38602)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532094425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_connection connected from Ok(127.0.0.1:59726)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532108603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_connection connected from Ok(127.0.0.1:39374)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532254814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.532964676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533002792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533033398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533064174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.53309301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533119377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533585309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533711344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.533744209Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534188134Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.53422278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534238528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534256896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534271374Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534285182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534353884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534374181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 123, 116, 116, 122, 118, 126, 122, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534389289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[0]R generation[0..12]: [80, 80, 79, 72, 72, 78, 74, 81, 78, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534403878Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534419296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 123, 116, 116, 122, 118, 126, 122, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534433534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[1]R generation[0..12]: [80, 80, 79, 72, 72, 78, 74, 81, 78, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534448092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.5344621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 123, 116, 116, 122, 118, 126, 122, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534476768Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[2]R generation[0..12]: [80, 80, 79, 72, 72, 78, 74, 81, 78, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534490597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534507645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534521113Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534535741Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534548969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534562028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"":"downstairs","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534575616Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534587724Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"0","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534618411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"1","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534633069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"client":"2","":"downstairs","session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534646497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534658746Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"c2dc649d-a790-4657-8199-5702242a6c22 is now active with session: 9eb644b3-7e7d-4b38-8d95-731446196393","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534671644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534683773Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510,"session_id":"9eb644b3-7e7d-4b38-8d95-731446196393"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:50.534696081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 36864 data: 11 11 11 12 11 7 4 5 6 02/30 Write block 34 len 36864 data: 11 11 9 6 6 7 6 4 6 03/30 Read block 584 len 8192 04/30 Read block 526 len 8192 05/30 Write block 263 len 24576 data: 8 9 9 7 8 6 06/30 Write block 463 len 20480 data: 4 5 5 6 5 07/30 Write block 475 len 36864 data: 7 7 8 9 9 9 9 8 9 08/30 Write block 518 len 24576 data: 8 8 7 9 7 7 09/30 Read block 489 len 36864 10/30 Read block 488 len 20480 11/30 Read block 104 len 12288 12/30 Write block 377 len 16384 data: 9 8 7 6 13/30 Write block 285 len 4096 data: 9 14/30 Read block 476 len 40960 15/30 Write block 83 len 32768 data: 7 8 9 11 7 8 8 6 16/30 Read block 133 len 4096 17/30 Flush 18/30 Write block 428 len 16384 data: 8 8 5 4 19/30 Write block 65 len 20480 data: 4 3 3 5 5 20/30 Flush 21/30 Write block 358 len 28672 data: 6 8 8 7 8 8 9 22/30 Read block 202 len 36864 23/30 Read block 387 len 36864 24/30 Write block 110 len 20480 data: 5 7 6 6 7 25/30 Write block 337 len 36864 data: 4 4 6 6 5 5 5 5 6 26/30 Read block 438 len 12288 27/30 Read block 591 len 8192 28/30 Read block 197 len 20480 29/30 Write block 36 len 40960 data: 10 7 7 8 7 5 7 9 8 9 30/30 Write block 145 len 24576 data: 5 5 5 8 10 8 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:50.554558993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 82 82 82 127 127 127 T T T 2 040-059 82 82 82 127 127 127 T T T 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 78 78 78 122 122 122 T T T 6 120-139 74 74 74 118 118 118 F F F 7 140-159 82 82 82 127 127 127 T T T 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 81 81 81 126 126 126 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 82 82 82 127 127 127 F F F 15 300-319 80 80 80 124 124 124 F F F 16 320-339 80 80 80 124 124 124 T T T 17 340-359 81 81 81 126 126 126 T T T 18 360-379 82 82 82 127 127 127 T T T 19 380-399 82 82 82 127 127 127 F F F 20 400-419 80 80 80 125 125 125 F F F 21 420-439 82 82 82 128 128 128 F F F 22 440-459 81 81 81 126 126 126 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 82 82 82 127 127 127 F F F 25 500-519 82 82 82 127 127 127 F F F 26 520-539 82 82 82 127 127 127 F F F 27 540-559 79 79 79 123 123 123 F F F 28 560-579 79 79 79 123 123 123 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 82, Max flush: 128 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 83 83 83 129 129 129 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 83 83 83 129 129 129 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 83 83 83 129 129 129 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 81 81 81 126 126 126 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 82 82 82 127 127 127 F F F 15 300-319 80 80 80 124 124 124 F F F 16 320-339 83 83 83 129 129 129 F F F 17 340-359 83 83 83 129 129 129 F F F 18 360-379 83 83 83 129 129 129 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 80 80 80 125 125 125 F F F 21 420-439 82 82 82 128 128 128 F F F 22 440-459 81 81 81 126 126 126 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 82 82 82 127 127 127 F F F 25 500-519 82 82 82 127 127 127 F F F 26 520-539 82 82 82 127 127 127 F F F 27 540-559 79 79 79 123 123 123 F F F 28 560-579 79 79 79 123 123 123 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 83, Max flush: 129 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.802699776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.847210015Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.84797713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.848029873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.8480541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.848078217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} {"msg":"Crucible f8b03473-4f57-4500-a7e8-2905ea1f4dcc has session id: 08dd5cb0-f2f2-49ff-81f4-937d573c35ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.848516043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Upstairs opts: Upstairs UUID: f8b03473-4f57-4500-a7e8-2905ea1f4dcc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.848559647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Crucible stats registered with UUID: f8b03473-4f57-4500-a7e8-2905ea1f4dcc","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.848577845Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:54.848591443Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849052426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849076923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} {"msg":"f8b03473-4f57-4500-a7e8-2905ea1f4dcc active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849137386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849155853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849185809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849213876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849239503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.84926422Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849287157Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849315463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849332711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849352009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:54348)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849385435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:57239)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849413101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:48785)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.849438128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850125183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850152789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850171787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850185565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850206813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850233849Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850921874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.85095118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.850976067Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.85151817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851548096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851564884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851582402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.85159655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851609918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851622267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851635625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[0]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 129, 118, 129, 127, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851649603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[0]R generation[0..12]: [80, 83, 83, 82, 82, 83, 74, 83, 82, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851663432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.8516775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[1]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 129, 118, 129, 127, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851691508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[1]R generation[0..12]: [80, 83, 83, 82, 82, 83, 74, 83, 82, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851705096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851719145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[2]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 129, 118, 129, 127, 112, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851732853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[2]R generation[0..12]: [80, 83, 83, 82, 82, 83, 74, 83, 82, 70, 64, 81]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851746581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.8517609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851776558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851790176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:43:54.851803274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851823492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"":"downstairs","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.85183684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851849678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"0","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851863367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"1","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851877025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"client":"2","":"downstairs","session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851890373Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851902672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"f8b03473-4f57-4500-a7e8-2905ea1f4dcc is now active with session: 08dd5cb0-f2f2-49ff-81f4-937d573c35ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.85191584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851928349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518,"session_id":"08dd5cb0-f2f2-49ff-81f4-937d573c35ca"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:54.851940937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 222 len 32768 data: 5 6 6 8 10 9 9 10 02/30 Write block 202 len 12288 data: 8 6 6 03/30 Flush 04/30 Write block 571 len 28672 data: 8 8 8 6 5 7 8 05/30 Flush 06/30 Flush 07/30 Write block 544 len 32768 data: 7 7 4 5 4 5 4 5 08/30 Read block 138 len 12288 09/30 Write block 489 len 36864 data: 7 7 6 6 8 7 10 10 8 10/30 Write block 215 len 16384 data: 7 8 6 4 11/30 Read block 111 len 28672 12/30 Read block 575 len 28672 13/30 Read block 191 len 24576 14/30 Read block 539 len 4096 15/30 Flush 16/30 Write block 113 len 12288 data: 7 8 8 17/30 Flush 18/30 Write block 414 len 32768 data: 9 9 8 9 7 6 6 9 19/30 Read block 261 len 12288 20/30 Read block 275 len 8192 21/30 Read block 76 len 32768 22/30 Read block 62 len 16384 23/30 Read block 153 len 16384 24/30 Write block 439 len 4096 data: 10 25/30 Write block 336 len 32768 data: 4 5 5 7 7 6 6 6 26/30 Write block 535 len 12288 data: 9 9 6 27/30 Read block 374 len 12288 28/30 Read block 78 len 8192 29/30 Write block 145 len 24576 data: 6 6 6 9 11 9 30/30 Write block 508 len 28672 data: 9 8 8 8 7 7 7 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:54.871741537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 83 83 83 129 129 129 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 83 83 83 129 129 129 T T T 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 84 84 84 130 130 130 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 82 82 82 127 127 127 F F F 15 300-319 80 80 80 124 124 124 F F F 16 320-339 83 83 83 129 129 129 T T T 17 340-359 83 83 83 129 129 129 T T T 18 360-379 83 83 83 129 129 129 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 80 80 80 125 125 125 T T T 21 420-439 82 82 82 128 128 128 T T T 22 440-459 81 81 81 126 126 126 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 84 84 84 133 133 133 F F F 25 500-519 82 82 82 127 127 127 T T T 26 520-539 82 82 82 127 127 127 T T T 27 540-559 84 84 84 133 133 133 F F F 28 560-579 84 84 84 131 131 131 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 84, Max flush: 134 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 124 124 124 F F F 1 020-039 83 83 83 129 129 129 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 74 74 74 118 118 118 F F F 7 140-159 85 85 85 135 135 135 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 84 84 84 130 130 130 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 82 82 82 127 127 127 F F F 15 300-319 80 80 80 124 124 124 F F F 16 320-339 85 85 85 135 135 135 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 83 83 83 129 129 129 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 85 85 85 135 135 135 F F F 21 420-439 85 85 85 135 135 135 F F F 22 440-459 81 81 81 126 126 126 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 84 84 84 133 133 133 F F F 25 500-519 85 85 85 135 135 135 F F F 26 520-539 85 85 85 135 135 135 F F F 27 540-559 84 84 84 133 133 133 F F F 28 560-579 84 84 84 131 131 131 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 85, Max flush: 135 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.125876565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:59.168651889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.169058269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.201004469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.201679355Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20171611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.201730798Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.201746187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} {"msg":"Crucible 506b82f6-438b-4c93-991f-706d3f4191e4 has session id: e4785094-7feb-4d17-aa7c-46f1a2943e63","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202168814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Upstairs opts: Upstairs UUID: 506b82f6-438b-4c93-991f-706d3f4191e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202214318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Crucible stats registered with UUID: 506b82f6-438b-4c93-991f-706d3f4191e4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202239025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:59.202254313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202681131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202717226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} {"msg":"506b82f6-438b-4c93-991f-706d3f4191e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202775139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202801236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202825462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20284114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202855799Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202873496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202897224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202922441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202947377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.202972704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:57719)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20300672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:62601)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203035486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:32874)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203062523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203797022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203835617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203865703Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20389385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203923686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.203953632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.204491675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.204523691Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.204540459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20510281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205130916Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205146244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205164202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20517804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205191459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205203657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205217135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[0]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 134, 118, 135, 127, 112, 133, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205231494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[0]R generation[0..12]: [80, 83, 83, 82, 82, 84, 74, 85, 82, 70, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205245512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20525975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[1]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 134, 118, 135, 127, 112, 133, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205273758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[1]R generation[0..12]: [80, 83, 83, 82, 82, 84, 74, 85, 82, 70, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205287607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205301725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[2]R flush_numbers[0..12]: [124, 129, 129, 128, 127, 134, 118, 135, 127, 112, 133, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205315723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[2]R generation[0..12]: [80, 83, 83, 82, 82, 84, 74, 85, 82, 70, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205329411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.20534348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205359008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205372586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205385764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205398713Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"":"downstairs","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:43:59.205412041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205430259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"0","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205443727Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"1","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205457175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"client":"2","":"downstairs","session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205470654Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205483052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"506b82f6-438b-4c93-991f-706d3f4191e4 is now active with session: e4785094-7feb-4d17-aa7c-46f1a2943e63","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205495861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205508069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526,"session_id":"e4785094-7feb-4d17-aa7c-46f1a2943e63"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:43:59.205520638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 32768 data: 10 9 8 9 9 10 9 9 02/30 Read block 186 len 20480 03/30 Flush 04/30 Write block 439 len 36864 data: 11 10 7 9 10 9 8 8 10 05/30 Write block 421 len 36864 data: 10 8 9 9 9 8 7 9 9 06/30 Read block 195 len 12288 07/30 Write block 502 len 36864 data: 9 11 10 9 10 10 11 10 10 08/30 Write block 573 len 28672 data: 9 7 6 8 9 6 5 09/30 Write block 133 len 28672 data: 7 7 7 6 5 5 5 10/30 Read block 16 len 4096 11/30 Write block 10 len 36864 data: 12 12 13 12 15 13 9 9 10 12/30 Write block 559 len 20480 data: 8 9 9 9 7 13/30 Read block 309 len 36864 14/30 Flush 15/30 Write block 1 len 36864 data: 4 5 7 7 9 11 12 13 13 16/30 Read block 361 len 12288 17/30 Write block 403 len 32768 data: 8 9 8 9 7 7 7 6 18/30 Read block 533 len 20480 19/30 Read block 322 len 4096 20/30 Read block 212 len 28672 21/30 Read block 294 len 24576 22/30 Write block 296 len 20480 data: 8 7 7 10 10 23/30 Flush 24/30 Write block 235 len 20480 data: 7 6 4 5 5 25/30 Read block 129 len 40960 26/30 Write block 492 len 24576 data: 7 9 8 11 11 9 27/30 Read block 319 len 20480 28/30 Write block 556 len 12288 data: 5 6 6 29/30 Write block 16 len 24576 data: 10 10 11 11 11 11 30/30 Write block 314 len 40960 data: 5 5 7 8 8 9 11 10 10 9 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:43:59.227024085Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 138 138 138 T T T 1 020-039 83 83 83 129 129 129 T T T 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 86 86 86 137 137 137 F F F 7 140-159 85 85 85 135 135 135 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 84 84 84 130 130 130 T T T 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 86 86 86 138 138 138 F F F 15 300-319 86 86 86 138 138 138 T T T 16 320-339 85 85 85 135 135 135 T T T 17 340-359 85 85 85 135 135 135 F F F 18 360-379 83 83 83 129 129 129 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 86 86 86 137 137 137 F F F 22 440-459 86 86 86 137 137 137 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 84 84 84 133 133 133 T T T 25 500-519 86 86 86 137 137 137 F F F 26 520-539 85 85 85 135 135 135 F F F 27 540-559 86 86 86 137 137 137 T T T 28 560-579 86 86 86 137 137 137 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 86, Max flush: 138 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 139 139 139 F F F 1 020-039 87 87 87 139 139 139 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 86 86 86 137 137 137 F F F 7 140-159 85 85 85 135 135 135 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 86 86 86 138 138 138 F F F 15 300-319 87 87 87 139 139 139 F F F 16 320-339 87 87 87 139 139 139 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 83 83 83 129 129 129 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 86 86 86 137 137 137 F F F 22 440-459 86 86 86 137 137 137 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 87 87 87 139 139 139 F F F 25 500-519 86 86 86 137 137 137 F F F 26 520-539 85 85 85 135 135 135 F F F 27 540-559 87 87 87 139 139 139 F F F 28 560-579 86 86 86 137 137 137 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 87, Max flush: 139 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.515408098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.560207061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561088621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561126577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561141555Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561156663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} {"msg":"Crucible ee6c29ff-a1cf-4145-a135-e370f21aef65 has session id: eb2559ab-49d2-4441-9c7a-4d58d6b9352d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561711784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Upstairs opts: Upstairs UUID: ee6c29ff-a1cf-4145-a135-e370f21aef65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56174259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Crucible stats registered with UUID: ee6c29ff-a1cf-4145-a135-e370f21aef65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.561762368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:03.561776236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562212572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562236029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} {"msg":"ee6c29ff-a1cf-4145-a135-e370f21aef65 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562297151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562315839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562344805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562371572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562398189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562425785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562452732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562482288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562512035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562540181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35948)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562565988Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36711)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562591205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57440)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.562695762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563170013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563208848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563241704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56327301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563345661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563378247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563878055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563909141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.563937467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564441115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564467241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56448175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56456271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564596645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564624312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56463824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564651919Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 129, 128, 127, 134, 137, 135, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564665947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[0]R generation[0..12]: [87, 87, 83, 82, 82, 84, 86, 85, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564679805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564693933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 129, 128, 127, 134, 137, 135, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564707942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[1]R generation[0..12]: [87, 87, 83, 82, 82, 84, 86, 85, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56472174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564735768Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 129, 128, 127, 134, 137, 135, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564749657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[2]R generation[0..12]: [87, 87, 83, 82, 82, 84, 86, 85, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564763465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564777423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.5647992Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Generation requested: 88 >= found:88","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-12-11T23:44:03.564831446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564854293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564868562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"":"downstairs","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.56488211Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564894348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"0","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564911126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"1","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564933604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"client":"2","":"downstairs","session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564959091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.564985297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"ee6c29ff-a1cf-4145-a135-e370f21aef65 is now active with session: eb2559ab-49d2-4441-9c7a-4d58d6b9352d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.565012004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.565028422Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534,"session_id":"eb2559ab-49d2-4441-9c7a-4d58d6b9352d"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:03.565058328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 24576 data: 12 10 8 10 12 12 02/30 Flush 03/30 Write block 366 len 24576 data: 10 9 8 10 10 10 04/30 Write block 10 len 28672 data: 13 13 14 13 16 14 11 05/30 Write block 152 len 4096 data: 11 06/30 Flush 07/30 Flush 08/30 Write block 506 len 28672 data: 11 11 12 11 11 9 8 09/30 Write block 557 len 12288 data: 7 7 9 10/30 Read block 552 len 16384 11/30 Read block 589 len 8192 12/30 Read block 231 len 28672 13/30 Read block 395 len 36864 14/30 Read block 460 len 28672 15/30 Read block 501 len 24576 16/30 Write block 281 len 12288 data: 6 9 8 17/30 Write block 130 len 8192 data: 7 6 18/30 Flush 19/30 Write block 429 len 32768 data: 10 6 5 7 7 8 10 9 20/30 Read block 346 len 8192 21/30 Read block 471 len 8192 22/30 Read block 444 len 32768 23/30 Read block 468 len 40960 24/30 Read block 304 len 8192 25/30 Read block 420 len 36864 26/30 Write block 362 len 20480 data: 9 9 10 9 11 27/30 Read block 220 len 24576 28/30 Write block 129 len 8192 data: 6 8 29/30 Read block 83 len 4096 30/30 Write block 567 len 16384 data: 5 7 5 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:03.583394249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 141 141 141 F F F 1 020-039 87 87 87 139 139 139 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 88 88 88 143 143 143 T T T 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 88 88 88 143 143 143 F F F 15 300-319 87 87 87 139 139 139 F F F 16 320-339 87 87 87 139 139 139 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 88 88 88 141 141 141 T T T 19 380-399 82 82 82 127 127 127 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 86 86 86 137 137 137 T T T 22 440-459 86 86 86 137 137 137 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 87 87 87 139 139 139 F F F 25 500-519 88 88 88 143 143 143 F F F 26 520-539 85 85 85 135 135 135 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 86 86 86 137 137 137 T T T 29 580-599 73 73 73 117 117 117 F F F Max gen: 88, Max flush: 143 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 141 141 141 F F F 1 020-039 87 87 87 139 139 139 F F F 2 040-059 83 83 83 129 129 129 F F F 3 060-079 82 82 82 128 128 128 F F F 4 080-099 82 82 82 127 127 127 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 89 89 89 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 F F F 10 200-219 84 84 84 133 133 133 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 F F F 14 280-299 88 88 88 143 143 143 F F F 15 300-319 87 87 87 139 139 139 F F F 16 320-339 87 87 87 139 139 139 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 89 89 89 144 144 144 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 89 89 89 144 144 144 F F F 22 440-459 86 86 86 137 137 137 F F F 23 460-479 82 82 82 127 127 127 F F F 24 480-499 87 87 87 139 139 139 F F F 25 500-519 88 88 88 143 143 143 F F F 26 520-539 85 85 85 135 135 135 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 73 73 73 117 117 117 F F F Max gen: 89, Max flush: 144 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.815730866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.859051133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.859675526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.859709651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.859723989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.859739558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} {"msg":"Crucible ba0b1d91-56c9-4217-9ee4-8492d0bf08e2 has session id: 75ce00ff-cf70-4afa-a90e-7206f38c2bf7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860242815Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Upstairs opts: Upstairs UUID: ba0b1d91-56c9-4217-9ee4-8492d0bf08e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.86027784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Crucible stats registered with UUID: ba0b1d91-56c9-4217-9ee4-8492d0bf08e2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860294218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:07.860312566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860840371Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860866498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} {"msg":"ba0b1d91-56c9-4217-9ee4-8492d0bf08e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.8609246Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860943628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860968995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.860995111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861021968Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861046975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861072342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861097269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861113547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861128345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56358)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861177109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62791)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861207605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64587)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861233402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861829488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861871322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861898909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.861923716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.862472888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.862605881Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863109619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863145304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863162482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.8631759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863189779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863209096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.86350181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.863521377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864018696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864493237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864519493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[0]R flush_numbers[0..12]: [141, 139, 129, 128, 127, 134, 144, 141, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864535262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[0]R generation[0..12]: [88, 87, 83, 82, 82, 84, 89, 88, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.86454973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864564198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[1]R flush_numbers[0..12]: [141, 139, 129, 128, 127, 134, 144, 141, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864578266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[1]R generation[0..12]: [88, 87, 83, 82, 82, 84, 89, 88, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864591955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864606133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[2]R flush_numbers[0..12]: [141, 139, 129, 128, 127, 134, 144, 141, 127, 112, 133, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864620161Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[2]R generation[0..12]: [88, 87, 83, 82, 82, 84, 89, 88, 82, 70, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864633649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864647608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864660706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864674004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864687173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864700221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"":"downstairs","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864721168Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864734107Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"0","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864747715Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"1","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864761303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"client":"2","":"downstairs","session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864774832Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.86478718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"ba0b1d91-56c9-4217-9ee4-8492d0bf08e2 is now active with session: 75ce00ff-cf70-4afa-a90e-7206f38c2bf7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864800058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864812317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542,"session_id":"75ce00ff-cf70-4afa-a90e-7206f38c2bf7"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:07.864824686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 4096 data: 8 02/30 Write block 64 len 36864 data: 7 5 4 4 6 6 8 10 8 03/30 Write block 584 len 8192 data: 3 3 04/30 Read block 269 len 32768 05/30 Read block 25 len 20480 06/30 Flush 07/30 Write block 537 len 28672 data: 7 6 8 6 6 7 8 08/30 Write block 231 len 12288 data: 9 7 6 09/30 Read block 417 len 32768 10/30 Read block 231 len 28672 11/30 Write block 435 len 40960 data: 11 10 8 10 12 11 8 10 11 10 12/30 Write block 36 len 32768 data: 11 8 8 9 8 6 8 10 13/30 Flush 14/30 Write block 309 len 4096 data: 6 15/30 Write block 198 len 16384 data: 9 8 7 8 16/30 Read block 385 len 20480 17/30 Write block 260 len 28672 data: 4 6 7 9 10 10 8 18/30 Write block 424 len 20480 data: 10 10 9 8 10 19/30 Read block 482 len 16384 20/30 Read block 472 len 32768 21/30 Write block 407 len 36864 data: 8 8 8 7 7 7 10 10 10 22/30 Write block 89 len 4096 data: 9 23/30 Read block 245 len 32768 24/30 Read block 5 len 20480 25/30 Read block 477 len 4096 26/30 Read block 108 len 16384 27/30 Read block 240 len 28672 28/30 Write block 481 len 8192 data: 10 9 29/30 Write block 14 len 40960 data: 17 15 12 11 12 12 12 12 13 13 30/30 Write block 208 len 12288 data: 7 8 9 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:07.883362122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 141 141 141 T T T 1 020-039 90 90 90 146 146 146 T T T 2 040-059 90 90 90 146 146 146 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 82 82 82 127 127 127 T T T 5 100-119 84 84 84 134 134 134 F F F 6 120-139 89 89 89 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 70 70 70 112 112 112 T T T 10 200-219 84 84 84 133 133 133 T T T 11 220-239 90 90 90 146 146 146 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 82 82 82 127 127 127 T T T 14 280-299 88 88 88 143 143 143 F F F 15 300-319 87 87 87 139 139 139 T T T 16 320-339 87 87 87 139 139 139 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 89 89 89 144 144 144 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 86 86 86 138 138 138 T T T 21 420-439 90 90 90 146 146 146 T T T 22 440-459 90 90 90 146 146 146 F F F 23 460-479 90 90 90 145 145 145 F F F 24 480-499 87 87 87 139 139 139 T T T 25 500-519 88 88 88 143 143 143 F F F 26 520-539 90 90 90 146 146 146 F F F 27 540-559 90 90 90 146 146 146 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 90, Max flush: 146 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 147 147 147 F F F 1 020-039 91 91 91 147 147 147 F F F 2 040-059 90 90 90 146 146 146 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 91 91 91 147 147 147 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 89 89 89 144 144 144 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 91 91 91 147 147 147 F F F 10 200-219 91 91 91 147 147 147 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 80 80 80 125 125 125 F F F 13 260-279 91 91 91 147 147 147 F F F 14 280-299 88 88 88 143 143 143 F F F 15 300-319 91 91 91 147 147 147 F F F 16 320-339 87 87 87 139 139 139 F F F 17 340-359 85 85 85 135 135 135 F F F 18 360-379 89 89 89 144 144 144 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 91 91 91 147 147 147 F F F 21 420-439 91 91 91 147 147 147 F F F 22 440-459 90 90 90 146 146 146 F F F 23 460-479 90 90 90 145 145 145 F F F 24 480-499 91 91 91 147 147 147 F F F 25 500-519 88 88 88 143 143 143 F F F 26 520-539 90 90 90 146 146 146 F F F 27 540-559 90 90 90 146 146 146 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 91, Max flush: 147 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.147405729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.191036807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.191707284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.191745029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.191760177Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.191776005Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} {"msg":"Crucible 77639e42-7f97-407e-9664-87db8c139642 has session id: 89bfad0b-341b-4a1b-b246-ba94e33368ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192280533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Upstairs opts: Upstairs UUID: 77639e42-7f97-407e-9664-87db8c139642, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192317158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Crucible stats registered with UUID: 77639e42-7f97-407e-9664-87db8c139642","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192334226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:22.192348414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.19286129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192886897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} {"msg":"77639e42-7f97-407e-9664-87db8c139642 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192954919Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.192973896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193001393Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.19302866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193058586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193086342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.19310418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193129697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193157164Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.19318723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:22.193220426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:22.193254402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:60219)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193284888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:22.193379056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:49644)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193408502Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.19343345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193805443Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193841779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193870315Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.193898482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.194561959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.194589386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195158415Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195188721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195204899Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195218697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195232646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:22.195246224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.194683664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:43370)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.194892509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.195814174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.19584234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.196375664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.196927856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.196966151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[0]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 144, 141, 127, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.196981749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[0]R generation[0..12]: [91, 91, 90, 90, 91, 84, 89, 88, 82, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.196996507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197011495Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[1]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 144, 141, 127, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197026313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[1]R generation[0..12]: [91, 91, 90, 90, 91, 84, 89, 88, 82, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197040722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.19705526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[2]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 144, 141, 127, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197069748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[2]R generation[0..12]: [91, 91, 90, 90, 91, 84, 89, 88, 82, 91, 91, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197083996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197102064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197115862Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197129471Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197142989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197156347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"":"downstairs","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197170206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"Wait for a query_work_queue command to finish before sending IO 89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197187243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"0","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197201202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"1","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197215Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"client":"2","":"downstairs","session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197228788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197241407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"77639e42-7f97-407e-9664-87db8c139642 is now active with session: 89bfad0b-341b-4a1b-b246-ba94e33368ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197254795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197267163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550,"session_id":"89bfad0b-341b-4a1b-b246-ba94e33368ca"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:32.197279802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 512 len 8192 data: 9 8 02/30 Write block 479 len 8192 data: 10 10 03/30 Write block 323 len 16384 data: 10 8 8 9 04/30 Read block 44 len 36864 05/30 Read block 218 len 4096 06/30 Write block 345 len 4096 data: 7 07/30 Write block 538 len 8192 data: 7 9 08/30 Read block 148 len 20480 09/30 Read block 295 len 24576 10/30 Write block 121 len 40960 data: 7 7 5 5 6 7 7 7 7 9 11/30 Flush 12/30 Write block 533 len 32768 data: 8 7 10 10 8 8 10 7 13/30 Write block 513 len 16384 data: 9 8 8 7 14/30 Write block 550 len 20480 data: 5 6 6 6 6 15/30 Write block 208 len 36864 data: 8 9 10 12 11 11 8 8 9 16/30 Read block 588 len 4096 17/30 Write block 250 len 20480 data: 8 5 5 7 6 18/30 Read block 139 len 4096 19/30 Write block 523 len 16384 data: 8 6 5 5 20/30 Write block 452 len 12288 data: 8 8 8 21/30 Read block 69 len 28672 22/30 Write block 549 len 16384 data: 6 6 7 7 23/30 Flush 24/30 Write block 362 len 12288 data: 10 10 11 25/30 Write block 510 len 16384 data: 12 10 10 10 26/30 Read block 69 len 12288 27/30 Read block 465 len 24576 28/30 Write block 476 len 8192 data: 8 9 29/30 Write block 277 len 28672 data: 5 5 5 4 7 10 9 30/30 Read block 79 len 24576 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:32.214827201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 147 147 147 F F F 1 020-039 91 91 91 147 147 147 F F F 2 040-059 90 90 90 146 146 146 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 91 91 91 147 147 147 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 92 92 92 148 148 148 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 91 91 91 147 147 147 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 92 92 92 149 149 149 F F F 13 260-279 91 91 91 147 147 147 T T T 14 280-299 88 88 88 143 143 143 T T T 15 300-319 91 91 91 147 147 147 F F F 16 320-339 92 92 92 148 148 148 F F F 17 340-359 92 92 92 148 148 148 F F F 18 360-379 89 89 89 144 144 144 T T T 19 380-399 82 82 82 127 127 127 F F F 20 400-419 91 91 91 147 147 147 F F F 21 420-439 91 91 91 147 147 147 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 92 92 92 148 148 148 T T T 24 480-499 92 92 92 148 148 148 F F F 25 500-519 92 92 92 149 149 149 T T T 26 520-539 92 92 92 149 149 149 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 92, Max flush: 149 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 147 147 147 F F F 1 020-039 91 91 91 147 147 147 F F F 2 040-059 90 90 90 146 146 146 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 91 91 91 147 147 147 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 92 92 92 148 148 148 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 82 82 82 127 127 127 F F F 9 180-199 91 91 91 147 147 147 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 92 92 92 149 149 149 F F F 13 260-279 93 93 93 150 150 150 F F F 14 280-299 93 93 93 150 150 150 F F F 15 300-319 91 91 91 147 147 147 F F F 16 320-339 92 92 92 148 148 148 F F F 17 340-359 92 92 92 148 148 148 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 82 82 82 127 127 127 F F F 20 400-419 91 91 91 147 147 147 F F F 21 420-439 91 91 91 147 147 147 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 93 93 93 150 150 150 F F F 24 480-499 92 92 92 148 148 148 F F F 25 500-519 93 93 93 150 150 150 F F F 26 520-539 92 92 92 149 149 149 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 93, Max flush: 150 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.453701096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.497097823Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.497872347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.497912362Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.497927151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.497942618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} {"msg":"Crucible 8c560124-b89b-4ac6-ba39-13d48de5a695 has session id: ba3a17de-ca7f-48dd-a07d-2319fde142db","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.498365296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Upstairs opts: Upstairs UUID: 8c560124-b89b-4ac6-ba39-13d48de5a695, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.49841721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Crucible stats registered with UUID: 8c560124-b89b-4ac6-ba39-13d48de5a695","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.498444856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:36.498466933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499030083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499062789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} {"msg":"8c560124-b89b-4ac6-ba39-13d48de5a695 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499129491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499154978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499178625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499202352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499224659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499245377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499327717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499368761Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499394118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499419625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_connection connected from Ok(127.0.0.1:57676)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499447342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_connection connected from Ok(127.0.0.1:43648)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.499590834Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_connection connected from Ok(127.0.0.1:41577)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.49961942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.50018615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500220746Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500249082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500278038Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500731172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500766198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.500883073Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.50090689Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501351755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.50147214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501497367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501512395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501526453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501544491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501558449Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501838045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501861711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[0]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 148, 141, 127, 147, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.50187633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[0]R generation[0..12]: [91, 91, 90, 90, 91, 84, 92, 88, 82, 91, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501890608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501905226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[1]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 148, 141, 127, 147, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501919304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[1]R generation[0..12]: [91, 91, 90, 90, 91, 84, 92, 88, 82, 91, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501933293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501947661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[2]R flush_numbers[0..12]: [147, 147, 146, 145, 147, 134, 148, 141, 127, 147, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501961839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[2]R generation[0..12]: [91, 91, 90, 90, 91, 84, 92, 88, 82, 91, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501975757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.501989776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502002954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502019982Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.50203337Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502046579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"":"downstairs","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502060187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502072605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"0","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502094982Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"1","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502108981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"client":"2","":"downstairs","session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502122599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502135477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"8c560124-b89b-4ac6-ba39-13d48de5a695 is now active with session: ba3a17de-ca7f-48dd-a07d-2319fde142db","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502148386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502160644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558,"session_id":"ba3a17de-ca7f-48dd-a07d-2319fde142db"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:36.502173213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 76 len 16384 data: 7 6 6 5 03/30 Read block 58 len 8192 04/30 Read block 487 len 28672 05/30 Write block 176 len 24576 data: 10 9 11 10 10 11 06/30 Write block 19 len 40960 data: 13 13 13 14 14 11 12 10 11 9 07/30 Read block 114 len 4096 08/30 Flush 09/30 Write block 335 len 24576 data: 4 5 6 6 8 8 10/30 Write block 563 len 8192 data: 8 9 11/30 Flush 12/30 Write block 293 len 20480 data: 6 7 8 9 8 13/30 Write block 183 len 36864 data: 10 10 8 7 6 7 5 5 6 14/30 Read block 14 len 40960 15/30 Read block 554 len 24576 16/30 Write block 499 len 8192 data: 9 10 17/30 Write block 555 len 40960 data: 5 6 8 8 10 10 10 10 9 10 18/30 Write block 462 len 32768 data: 7 5 6 6 7 6 7 7 19/30 Read block 570 len 16384 20/30 Read block 344 len 28672 21/30 Write block 256 len 8192 data: 7 6 22/30 Write block 544 len 16384 data: 8 8 5 6 23/30 Write block 526 len 8192 data: 6 6 24/30 Flush 25/30 Write block 390 len 36864 data: 6 7 6 7 7 11 11 11 11 26/30 Read block 73 len 28672 27/30 Write block 295 len 24576 data: 9 10 9 8 11 11 28/30 Read block 35 len 24576 29/30 Read block 364 len 8192 30/30 Write block 574 len 16384 data: 8 7 9 10 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:36.521699986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 152 152 152 F F F 1 020-039 94 94 94 152 152 152 F F F 2 040-059 90 90 90 146 146 146 F F F 3 060-079 94 94 94 152 152 152 F F F 4 080-099 91 91 91 147 147 147 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 92 92 92 148 148 148 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 94 94 94 152 152 152 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 93 93 93 150 150 150 F F F 14 280-299 94 94 94 154 154 154 T T T 15 300-319 91 91 91 147 147 147 T T T 16 320-339 94 94 94 153 153 153 F F F 17 340-359 94 94 94 153 153 153 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 82 82 82 127 127 127 T T T 20 400-419 91 91 91 147 147 147 F F F 21 420-439 91 91 91 147 147 147 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 94 94 94 154 154 154 F F F 24 480-499 94 94 94 154 154 154 F F F 25 500-519 94 94 94 154 154 154 F F F 26 520-539 94 94 94 154 154 154 F F F 27 540-559 94 94 94 154 154 154 F F F 28 560-579 94 94 94 154 154 154 T T T 29 580-599 90 90 90 145 145 145 F F F Max gen: 94, Max flush: 154 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 152 152 152 F F F 1 020-039 94 94 94 152 152 152 F F F 2 040-059 90 90 90 146 146 146 F F F 3 060-079 94 94 94 152 152 152 F F F 4 080-099 91 91 91 147 147 147 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 92 92 92 148 148 148 F F F 7 140-159 88 88 88 141 141 141 F F F 8 160-179 94 94 94 152 152 152 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 93 93 93 150 150 150 F F F 14 280-299 95 95 95 155 155 155 F F F 15 300-319 95 95 95 155 155 155 F F F 16 320-339 94 94 94 153 153 153 F F F 17 340-359 94 94 94 153 153 153 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 95 95 95 155 155 155 F F F 20 400-419 91 91 91 147 147 147 F F F 21 420-439 91 91 91 147 147 147 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 94 94 94 154 154 154 F F F 24 480-499 94 94 94 154 154 154 F F F 25 500-519 94 94 94 154 154 154 F F F 26 520-539 94 94 94 154 154 154 F F F 27 540-559 94 94 94 154 154 154 F F F 28 560-579 95 95 95 155 155 155 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 95, Max flush: 155 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.762689518Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:40.804904292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.805314661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.837091732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.837811833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.837850628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.837865446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.837880654Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} {"msg":"Crucible 517bdcdb-ca35-4d0a-aace-0c9d24e137ec has session id: ba0ae163-820d-4cd2-97f2-aa3d35b74b48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.838308451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Upstairs opts: Upstairs UUID: 517bdcdb-ca35-4d0a-aace-0c9d24e137ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.838350426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Crucible stats registered with UUID: 517bdcdb-ca35-4d0a-aace-0c9d24e137ec","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.838382482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:40.838406239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.838973328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839006114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} {"msg":"517bdcdb-ca35-4d0a-aace-0c9d24e137ec active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839024292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.83904388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839071956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839088704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839114551Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839143087Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839172344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.83919872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839227037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839258503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:44048)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839303847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:55196)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839345322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:50345)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.839389946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840030777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840060953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840077601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840092579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840206055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840238441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840712732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840743228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.840760696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841345034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.84137348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841388378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841400896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841418494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841432433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841444961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841458959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[0]R flush_numbers[0..12]: [152, 152, 146, 152, 147, 134, 148, 141, 152, 154, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841473387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[0]R generation[0..12]: [94, 94, 90, 94, 91, 84, 92, 88, 94, 94, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841487516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841501664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[1]R flush_numbers[0..12]: [152, 152, 146, 152, 147, 134, 148, 141, 152, 154, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841515592Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[1]R generation[0..12]: [94, 94, 90, 94, 91, 84, 92, 88, 94, 94, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841529171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841543399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[2]R flush_numbers[0..12]: [152, 152, 146, 152, 147, 134, 148, 141, 152, 154, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841558017Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[2]R generation[0..12]: [94, 94, 90, 94, 91, 84, 92, 88, 94, 94, 92, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841572055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841586184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841599492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.84161271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841628468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841649736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"":"downstairs","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841663344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841675453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"0","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841689041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"1","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841702319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"client":"2","":"downstairs","session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841716027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841728376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"517bdcdb-ca35-4d0a-aace-0c9d24e137ec is now active with session: ba0ae163-820d-4cd2-97f2-aa3d35b74b48","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841744384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841767671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566,"session_id":"ba0ae163-820d-4cd2-97f2-aa3d35b74b48"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:40.841793608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 322 len 32768 data: 11 11 9 9 10 10 8 7 02/30 Read block 561 len 8192 03/30 Write block 206 len 12288 data: 6 8 9 04/30 Flush 05/30 Write block 419 len 20480 data: 7 7 11 9 10 06/30 Write block 122 len 12288 data: 8 6 6 07/30 Write block 331 len 24576 data: 4 4 4 4 5 6 08/30 Read block 77 len 28672 09/30 Write block 472 len 28672 data: 9 8 7 9 9 10 10 10/30 Write block 403 len 8192 data: 9 10 11/30 Read block 482 len 24576 12/30 Read block 520 len 16384 13/30 Read block 414 len 4096 14/30 Read block 430 len 36864 15/30 Write block 163 len 32768 data: 5 8 8 7 7 6 6 5 16/30 Flush 17/30 Write block 295 len 24576 data: 10 11 10 9 12 12 18/30 Write block 59 len 24576 data: 9 7 10 10 7 8 19/30 Read block 462 len 16384 20/30 Read block 503 len 24576 21/30 Read block 95 len 40960 22/30 Write block 141 len 4096 data: 5 23/30 Read block 143 len 12288 24/30 Read block 553 len 16384 25/30 Read block 99 len 28672 26/30 Read block 258 len 36864 27/30 Write block 349 len 12288 data: 6 6 7 28/30 Read block 533 len 4096 29/30 Write block 94 len 4096 data: 2 30/30 Write block 152 len 16384 data: 12 13 13 12 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Sent Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:40.861554882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 152 152 152 F F F 1 020-039 94 94 94 152 152 152 F F F 2 040-059 90 90 90 146 146 146 T T T 3 060-079 94 94 94 152 152 152 T T T 4 080-099 91 91 91 147 147 147 T T T 5 100-119 84 84 84 134 134 134 F F F 6 120-139 96 96 96 157 157 157 F F F 7 140-159 88 88 88 141 141 141 T T T 8 160-179 96 96 96 157 157 157 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 96 96 96 156 156 156 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 93 93 93 150 150 150 F F F 14 280-299 95 95 95 155 155 155 T T T 15 300-319 95 95 95 155 155 155 T T T 16 320-339 96 96 96 157 157 157 F F F 17 340-359 94 94 94 153 153 153 T T T 18 360-379 93 93 93 150 150 150 F F F 19 380-399 95 95 95 155 155 155 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 96 96 96 157 157 157 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 96 96 96 157 157 157 F F F 24 480-499 94 94 94 154 154 154 F F F 25 500-519 94 94 94 154 154 154 F F F 26 520-539 94 94 94 154 154 154 F F F 27 540-559 94 94 94 154 154 154 F F F 28 560-579 95 95 95 155 155 155 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 96, Max flush: 157 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 152 152 152 F F F 1 020-039 94 94 94 152 152 152 F F F 2 040-059 97 97 97 158 158 158 F F F 3 060-079 97 97 97 158 158 158 F F F 4 080-099 97 97 97 158 158 158 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 96 96 96 157 157 157 F F F 7 140-159 97 97 97 158 158 158 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 96 96 96 156 156 156 F F F 11 220-239 90 90 90 146 146 146 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 93 93 93 150 150 150 F F F 14 280-299 97 97 97 158 158 158 F F F 15 300-319 97 97 97 158 158 158 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 97 97 97 158 158 158 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 95 95 95 155 155 155 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 96 96 96 157 157 157 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 96 96 96 157 157 157 F F F 24 480-499 94 94 94 154 154 154 F F F 25 500-519 94 94 94 154 154 154 F F F 26 520-539 94 94 94 154 154 154 F F F 27 540-559 94 94 94 154 154 154 F F F 28 560-579 95 95 95 155 155 155 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 97, Max flush: 158 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.146881774Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.191036158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.1918189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.191877273Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.191894411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.191910469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} {"msg":"Crucible 1e648b6b-27e3-4bbe-ba05-e2d8a0584b17 has session id: 8105fd23-43ec-4fc1-915a-615ba2c625e0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.192329887Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Upstairs opts: Upstairs UUID: 1e648b6b-27e3-4bbe-ba05-e2d8a0584b17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.192374831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Crucible stats registered with UUID: 1e648b6b-27e3-4bbe-ba05-e2d8a0584b17","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.192405277Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:45.192426245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193041128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193068565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} {"msg":"1e648b6b-27e3-4bbe-ba05-e2d8a0584b17 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193128738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193152124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.19318466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193203658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193219116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193232945Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193248033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193274839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193344031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193373347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36316)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193518099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45952)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193664121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57145)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.193694387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194184986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194216772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.19423267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194246628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194734368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194763194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.194792431Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195045439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195336383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195373948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195402275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195430721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195586602Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195616748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195632526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195907172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.19592845Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[0]R flush_numbers[0..12]: [152, 152, 158, 158, 158, 134, 157, 158, 157, 154, 156, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195945787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[0]R generation[0..12]: [94, 94, 97, 97, 97, 84, 96, 97, 96, 94, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195960206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195974214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[1]R flush_numbers[0..12]: [152, 152, 158, 158, 158, 134, 157, 158, 157, 154, 156, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.195988332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[1]R generation[0..12]: [94, 94, 97, 97, 97, 84, 96, 97, 96, 94, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.19600201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196015969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[2]R flush_numbers[0..12]: [152, 152, 158, 158, 158, 134, 157, 158, 157, 154, 156, 146]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196029717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[2]R generation[0..12]: [94, 94, 97, 97, 97, 84, 96, 97, 96, 94, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196043295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196057203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196070392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.19608372Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196096809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196113196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"":"downstairs","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196127875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196140153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"0","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196153462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"1","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196173189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"client":"2","":"downstairs","session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196186587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196199056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"1e648b6b-27e3-4bbe-ba05-e2d8a0584b17 is now active with session: 8105fd23-43ec-4fc1-915a-615ba2c625e0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196211904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196224153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574,"session_id":"8105fd23-43ec-4fc1-915a-615ba2c625e0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:45.196236481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 28672 data: 7 5 6 9 9 8 8 02/30 Read block 517 len 28672 03/30 Read block 303 len 16384 04/30 Read block 492 len 12288 05/30 Read block 594 len 20480 06/30 Flush 07/30 Write block 395 len 16384 data: 12 12 12 12 08/30 Write block 475 len 28672 data: 10 10 11 11 11 11 11 09/30 Write block 41 len 28672 data: 7 9 11 9 10 8 8 10/30 Read block 257 len 12288 11/30 Write block 428 len 32768 data: 11 11 7 6 8 8 9 12 12/30 Read block 171 len 36864 13/30 Read block 245 len 16384 14/30 Write block 7 len 8192 data: 13 14 15/30 Write block 304 len 12288 data: 7 6 6 16/30 Read block 166 len 28672 17/30 Write block 509 len 20480 data: 12 13 11 11 11 18/30 Read block 298 len 32768 19/30 Write block 218 len 28672 data: 5 3 4 5 6 7 7 20/30 Read block 589 len 16384 21/30 Flush 22/30 Flush 23/30 Write block 528 len 36864 data: 4 4 4 7 8 9 8 11 11 24/30 Write block 273 len 32768 data: 7 7 8 7 6 6 6 5 25/30 Write block 575 len 12288 data: 8 10 11 26/30 Read block 571 len 12288 27/30 Read block 237 len 4096 28/30 Write block 127 len 20480 data: 8 8 8 10 7 29/30 Write block 22 len 36864 data: 15 15 12 13 11 12 10 10 10 30/30 Read block 34 len 32768 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:45.21781582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 94 94 94 152 152 152 T T T 2 040-059 98 98 98 160 160 160 F F F 3 060-079 97 97 97 158 158 158 F F F 4 080-099 97 97 97 158 158 158 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 96 96 96 157 157 157 T T T 7 140-159 97 97 97 158 158 158 F F F 8 160-179 98 98 98 159 159 159 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 93 93 93 150 150 150 T T T 14 280-299 97 97 97 158 158 158 T T T 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 97 97 97 158 158 158 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 98 98 98 160 160 160 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 98 98 98 160 160 160 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 94 94 94 154 154 154 T T T 27 540-559 94 94 94 154 154 154 F F F 28 560-579 95 95 95 155 155 155 T T T 29 580-599 90 90 90 145 145 145 F F F Max gen: 98, Max flush: 160 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 99 99 99 161 161 161 F F F 2 040-059 98 98 98 160 160 160 F F F 3 060-079 97 97 97 158 158 158 F F F 4 080-099 97 97 97 158 158 158 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 97 97 97 158 158 158 F F F 8 160-179 98 98 98 159 159 159 F F F 9 180-199 94 94 94 154 154 154 F F F 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 99 99 99 161 161 161 F F F 14 280-299 99 99 99 161 161 161 F F F 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 97 97 97 158 158 158 F F F 18 360-379 93 93 93 150 150 150 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 98 98 98 160 160 160 F F F 22 440-459 92 92 92 149 149 149 F F F 23 460-479 98 98 98 160 160 160 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 94 94 94 154 154 154 F F F 28 560-579 99 99 99 161 161 161 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 99, Max flush: 161 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.468066111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.512742229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.513608821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.513650286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.513666034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.513681983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} {"msg":"Crucible 098a2fa1-9315-4e2b-b62a-db76dad1ce5b has session id: 2e224d19-1a04-4324-afe1-cba12e1b08bc","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.514217376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Upstairs opts: Upstairs UUID: 098a2fa1-9315-4e2b-b62a-db76dad1ce5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.514260691Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Crucible stats registered with UUID: 098a2fa1-9315-4e2b-b62a-db76dad1ce5b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.514285847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:49.514308754Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.51490646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.514940586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} {"msg":"098a2fa1-9315-4e2b-b62a-db76dad1ce5b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.51499362Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515021096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515048313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515073019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515103126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515132662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.51515346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515175837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515239349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515286613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:58513)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515324418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38803)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515356744Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:39039)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515384461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515861552Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515899687Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.515975578Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516001714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516342032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516387116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516422482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516441309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.516979343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517009259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517025067Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517038955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517115036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517135933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517150132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517579068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517599156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[0]R flush_numbers[0..12]: [160, 161, 160, 158, 158, 134, 161, 158, 159, 154, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517617743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[0]R generation[0..12]: [98, 99, 98, 97, 97, 84, 99, 97, 98, 94, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517632732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.51764705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[1]R flush_numbers[0..12]: [160, 161, 160, 158, 158, 134, 161, 158, 159, 154, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517661118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[1]R generation[0..12]: [98, 99, 98, 97, 97, 84, 99, 97, 98, 94, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517674736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517688685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[2]R flush_numbers[0..12]: [160, 161, 160, 158, 158, 134, 161, 158, 159, 154, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517702553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[2]R generation[0..12]: [98, 99, 98, 97, 97, 84, 99, 97, 98, 94, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517716131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517730779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517744068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517757476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517770444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:44:49.517783583Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"":"downstairs","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517800781Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517816219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"0","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517830007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"1","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517843255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"client":"2","":"downstairs","session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517856674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517869102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"098a2fa1-9315-4e2b-b62a-db76dad1ce5b is now active with session: 2e224d19-1a04-4324-afe1-cba12e1b08bc","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517882001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517894109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582,"session_id":"2e224d19-1a04-4324-afe1-cba12e1b08bc"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:49.517906567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 4096 data: 9 02/30 Read block 104 len 12288 03/30 Write block 268 len 8192 data: 7 6 04/30 Read block 442 len 36864 05/30 Read block 90 len 28672 06/30 Read block 384 len 8192 07/30 Flush 08/30 Write block 47 len 28672 data: 9 10 8 9 9 8 10 09/30 Flush 10/30 Write block 292 len 32768 data: 6 7 8 11 12 11 10 13 11/30 Read block 403 len 36864 12/30 Read block 204 len 40960 13/30 Write block 552 len 8192 data: 8 7 14/30 Write block 79 len 8192 data: 6 6 15/30 Read block 336 len 40960 16/30 Read block 36 len 32768 17/30 Read block 214 len 4096 18/30 Read block 265 len 16384 19/30 Write block 367 len 16384 data: 10 9 11 11 20/30 Write block 180 len 4096 data: 11 21/30 Write block 381 len 4096 data: 6 22/30 Write block 492 len 32768 data: 8 10 9 12 12 10 8 10 23/30 Write block 570 len 8192 data: 7 9 24/30 Write block 14 len 24576 data: 18 16 13 12 13 14 25/30 Read block 493 len 40960 26/30 Read block 561 len 4096 27/30 Write block 453 len 12288 data: 9 9 7 28/30 Read block 368 len 16384 29/30 Write block 434 len 24576 data: 10 13 11 9 11 13 30/30 Write block 18 len 40960 data: 14 15 14 14 16 16 13 14 12 13 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 8 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:49.535717844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 T T T 1 020-039 99 99 99 161 161 161 T T T 2 040-059 100 100 100 163 163 163 F F F 3 060-079 97 97 97 158 158 158 T T T 4 080-099 97 97 97 158 158 158 T T T 5 100-119 84 84 84 134 134 134 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 97 97 97 158 158 158 F F F 8 160-179 98 98 98 159 159 159 F F F 9 180-199 94 94 94 154 154 154 T T T 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 100 100 100 162 162 162 F F F 14 280-299 99 99 99 161 161 161 T T T 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 97 97 97 158 158 158 F F F 18 360-379 93 93 93 150 150 150 T T T 19 380-399 98 98 98 160 160 160 T T T 20 400-419 96 96 96 157 157 157 F F F 21 420-439 98 98 98 160 160 160 T T T 22 440-459 92 92 92 149 149 149 T T T 23 460-479 98 98 98 160 160 160 F F F 24 480-499 98 98 98 160 160 160 T T T 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 94 94 94 154 154 154 T T T 28 560-579 99 99 99 161 161 161 T T T 29 580-599 90 90 90 145 145 145 F F F Max gen: 100, Max flush: 163 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 164 164 164 F F F 1 020-039 101 101 101 164 164 164 F F F 2 040-059 100 100 100 163 163 163 F F F 3 060-079 101 101 101 164 164 164 F F F 4 080-099 101 101 101 164 164 164 F F F 5 100-119 84 84 84 134 134 134 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 97 97 97 158 158 158 F F F 8 160-179 98 98 98 159 159 159 F F F 9 180-199 101 101 101 164 164 164 F F F 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 94 94 94 154 154 154 F F F 13 260-279 100 100 100 162 162 162 F F F 14 280-299 101 101 101 164 164 164 F F F 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 97 97 97 158 158 158 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 101 101 101 164 164 164 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 101 101 101 164 164 164 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 98 98 98 160 160 160 F F F 24 480-499 101 101 101 164 164 164 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 101 101 101 164 164 164 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 101, Max flush: 164 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.826469652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.869977466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.870677859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.870715224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.870729882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87074503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} {"msg":"Crucible c94d64b8-6db5-4dbe-b25e-8702e5bb3d77 has session id: 32ba5908-c061-45c1-b56f-e439a87eacb4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871261036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Upstairs opts: Upstairs UUID: c94d64b8-6db5-4dbe-b25e-8702e5bb3d77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871305001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Crucible stats registered with UUID: c94d64b8-6db5-4dbe-b25e-8702e5bb3d77","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871332237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:53.871357054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.8717933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871826416Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} {"msg":"c94d64b8-6db5-4dbe-b25e-8702e5bb3d77 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871883949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871922364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871960149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.871986526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872013193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87203829Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872057407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872072595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872086733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872148826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45785)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872251693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41793)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872287568Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42263)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872389396Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872852848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872897253Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872927469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.872950196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.873006889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.873036046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.873491789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.873617323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.873690394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874051819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874082265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874099303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87412464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874139698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874153237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874218499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874238756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[0]R flush_numbers[0..12]: [164, 164, 163, 164, 164, 134, 161, 158, 159, 164, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874256664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[0]R generation[0..12]: [101, 101, 100, 101, 101, 84, 99, 97, 98, 101, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874271132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87428519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[1]R flush_numbers[0..12]: [164, 164, 163, 164, 164, 134, 161, 158, 159, 164, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874299249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[1]R generation[0..12]: [101, 101, 100, 101, 101, 84, 99, 97, 98, 101, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874313117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874327215Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[2]R flush_numbers[0..12]: [164, 164, 163, 164, 164, 134, 161, 158, 159, 164, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874340893Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[2]R generation[0..12]: [101, 101, 100, 101, 101, 84, 99, 97, 98, 101, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874354582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87436866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874381818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874395306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874408375Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874421463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"":"downstairs","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874435072Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.8744492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"0","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874471477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"1","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874484975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"client":"2","":"downstairs","session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874498244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874510712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"c94d64b8-6db5-4dbe-b25e-8702e5bb3d77 is now active with session: 32ba5908-c061-45c1-b56f-e439a87eacb4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.87452377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874536029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590,"session_id":"32ba5908-c061-45c1-b56f-e439a87eacb4"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:53.874548407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 347 len 8192 data: 6 7 02/30 Read block 180 len 40960 03/30 Read block 15 len 8192 04/30 Read block 473 len 4096 05/30 Flush 06/30 Write block 344 len 28672 data: 6 8 6 7 8 7 7 07/30 Read block 435 len 8192 08/30 Read block 430 len 28672 09/30 Read block 208 len 28672 10/30 Write block 551 len 20480 data: 8 9 8 7 6 11/30 Flush 12/30 Write block 43 len 36864 data: 12 10 11 10 10 11 9 10 10 13/30 Read block 474 len 20480 14/30 Read block 168 len 16384 15/30 Read block 567 len 20480 16/30 Flush 17/30 Write block 156 len 28672 data: 13 12 8 5 6 8 6 18/30 Read block 353 len 8192 19/30 Write block 257 len 24576 data: 7 5 4 5 7 8 20/30 Write block 388 len 36864 data: 6 6 7 8 7 8 8 13 13 21/30 Read block 564 len 24576 22/30 Read block 425 len 32768 23/30 Read block 126 len 4096 24/30 Read block 360 len 28672 25/30 Read block 568 len 24576 26/30 Write block 172 len 36864 data: 4 5 7 5 11 10 12 11 12 27/30 Write block 77 len 36864 data: 7 7 7 7 6 8 8 9 10 28/30 Write block 112 len 24576 data: 7 8 9 9 9 7 29/30 Write block 36 len 20480 data: 12 9 9 10 9 30/30 Read block 343 len 32768 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:53.89505183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 164 164 164 F F F 1 020-039 101 101 101 164 164 164 T T T 2 040-059 102 102 102 167 167 167 T T T 3 060-079 101 101 101 164 164 164 T T T 4 080-099 101 101 101 164 164 164 T T T 5 100-119 84 84 84 134 134 134 T T T 6 120-139 99 99 99 161 161 161 F F F 7 140-159 97 97 97 158 158 158 T T T 8 160-179 98 98 98 159 159 159 T T T 9 180-199 101 101 101 164 164 164 T T T 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 94 94 94 154 154 154 T T T 13 260-279 100 100 100 162 162 162 T T T 14 280-299 101 101 101 164 164 164 F F F 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 102 102 102 166 166 166 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 101 101 101 164 164 164 T T T 20 400-419 96 96 96 157 157 157 F F F 21 420-439 101 101 101 164 164 164 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 98 98 98 160 160 160 F F F 24 480-499 101 101 101 164 164 164 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 102, Max flush: 167 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 164 164 164 F F F 1 020-039 103 103 103 168 168 168 F F F 2 040-059 103 103 103 168 168 168 F F F 3 060-079 103 103 103 168 168 168 F F F 4 080-099 103 103 103 168 168 168 F F F 5 100-119 103 103 103 168 168 168 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 103 103 103 168 168 168 F F F 8 160-179 103 103 103 168 168 168 F F F 9 180-199 103 103 103 168 168 168 F F F 10 200-219 98 98 98 160 160 160 F F F 11 220-239 98 98 98 160 160 160 F F F 12 240-259 103 103 103 168 168 168 F F F 13 260-279 103 103 103 168 168 168 F F F 14 280-299 101 101 101 164 164 164 F F F 15 300-319 98 98 98 160 160 160 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 102 102 102 166 166 166 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 101 101 101 164 164 164 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 98 98 98 160 160 160 F F F 24 480-499 101 101 101 164 164 164 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 90 90 90 145 145 145 F F F Max gen: 103, Max flush: 168 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.165781286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:58.209116701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.209576333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.241654437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24251539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.242556305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.242571233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.242586241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} {"msg":"Crucible 15c33c74-9b73-4c0f-8d2d-ba2a38c7736a has session id: cc980b9d-9d7d-4a81-bfdb-af74862a9a85","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24307938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Upstairs opts: Upstairs UUID: 15c33c74-9b73-4c0f-8d2d-ba2a38c7736a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243114846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Crucible stats registered with UUID: 15c33c74-9b73-4c0f-8d2d-ba2a38c7736a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243131014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:58.243144402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243617433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24364361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} {"msg":"15c33c74-9b73-4c0f-8d2d-ba2a38c7736a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243705852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.2437251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243748877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243764945Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243784402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243810549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243838256Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243866642Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243894669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243923425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:59575)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243952821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:64970)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.243984428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:50933)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244025372Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244618689Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244654014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244669663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244683411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244886836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.244907203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.245393522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.245421889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.245437107Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24597497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246004157Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246018755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246031363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246044961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24605819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246070588Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246087826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[0]R flush_numbers[0..12]: [164, 168, 168, 168, 168, 168, 161, 168, 168, 168, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246102205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[0]R generation[0..12]: [101, 103, 103, 103, 103, 103, 99, 103, 103, 103, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246116043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246130261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[1]R flush_numbers[0..12]: [164, 168, 168, 168, 168, 168, 161, 168, 168, 168, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246144139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[1]R generation[0..12]: [101, 103, 103, 103, 103, 103, 99, 103, 103, 103, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246157958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246171906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[2]R flush_numbers[0..12]: [164, 168, 168, 168, 168, 168, 161, 168, 168, 168, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246185664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[2]R generation[0..12]: [101, 103, 103, 103, 103, 103, 99, 103, 103, 103, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246199412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246213531Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246226709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246240007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246253116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246266074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"":"downstairs","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-11T23:44:58.246291661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246304409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"0","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246317728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"1","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246330906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"client":"2","":"downstairs","session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246344064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246356643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"15c33c74-9b73-4c0f-8d2d-ba2a38c7736a is now active with session: cc980b9d-9d7d-4a81-bfdb-af74862a9a85","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246369491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.24638147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598,"session_id":"cc980b9d-9d7d-4a81-bfdb-af74862a9a85"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:44:58.246393748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 65 len 16384 data: 6 5 5 7 02/30 Write block 42 len 12288 data: 10 13 11 03/30 Write block 296 len 20480 data: 13 12 11 14 13 04/30 Write block 54 len 12288 data: 7 7 6 05/30 Read block 510 len 12288 06/30 Read block 305 len 8192 07/30 Write block 150 len 36864 data: 11 9 13 14 14 13 14 13 9 08/30 Write block 87 len 24576 data: 8 9 10 7 6 4 09/30 Flush 10/30 Write block 278 len 12288 data: 7 7 6 11/30 Flush 12/30 Flush 13/30 Write block 196 len 28672 data: 8 8 10 9 8 9 9 14/30 Write block 186 len 4096 data: 8 15/30 Write block 478 len 24576 data: 12 12 12 12 10 10 16/30 Read block 261 len 28672 17/30 Write block 236 len 20480 data: 7 5 6 6 6 18/30 Write block 400 len 16384 data: 11 11 11 10 19/30 Read block 398 len 4096 20/30 Write block 144 len 16384 data: 4 7 7 7 21/30 Read block 481 len 36864 22/30 Read block 422 len 24576 23/30 Read block 204 len 4096 24/30 Write block 220 len 12288 data: 5 6 7 25/30 Write block 589 len 28672 data: 6 7 8 7 8 9 7 26/30 Read block 407 len 36864 27/30 Read block 419 len 40960 28/30 Read block 81 len 4096 29/30 Write block 200 len 4096 data: 9 30/30 Write block 6 len 36864 data: 12 14 15 14 14 14 15 14 19 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:44:58.264096538Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 164 164 164 T T T 1 020-039 103 103 103 168 168 168 F F F 2 040-059 104 104 104 169 169 169 F F F 3 060-079 104 104 104 169 169 169 F F F 4 080-099 104 104 104 169 169 169 F F F 5 100-119 103 103 103 168 168 168 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 104 104 104 169 169 169 T T T 8 160-179 103 103 103 168 168 168 F F F 9 180-199 103 103 103 168 168 168 T T T 10 200-219 98 98 98 160 160 160 T T T 11 220-239 98 98 98 160 160 160 T T T 12 240-259 103 103 103 168 168 168 T T T 13 260-279 104 104 104 170 170 170 F F F 14 280-299 104 104 104 170 170 170 F F F 15 300-319 104 104 104 169 169 169 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 102 102 102 166 166 166 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 96 96 96 157 157 157 T T T 21 420-439 101 101 101 164 164 164 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 98 98 98 160 160 160 T T T 24 480-499 101 101 101 164 164 164 T T T 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 90 90 90 145 145 145 T T T Max gen: 104, Max flush: 170 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 171 171 171 F F F 1 020-039 103 103 103 168 168 168 F F F 2 040-059 104 104 104 169 169 169 F F F 3 060-079 104 104 104 169 169 169 F F F 4 080-099 104 104 104 169 169 169 F F F 5 100-119 103 103 103 168 168 168 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 105 105 105 171 171 171 F F F 8 160-179 103 103 103 168 168 168 F F F 9 180-199 105 105 105 171 171 171 F F F 10 200-219 105 105 105 171 171 171 F F F 11 220-239 105 105 105 171 171 171 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 104 104 104 170 170 170 F F F 15 300-319 104 104 104 169 169 169 F F F 16 320-339 96 96 96 157 157 157 F F F 17 340-359 102 102 102 166 166 166 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 105 105 105 171 171 171 F F F 21 420-439 101 101 101 164 164 164 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 105 105 105 171 171 171 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 105 105 105 171 171 171 F F F Max gen: 105, Max flush: 171 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.570345961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.615111568Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.615877642Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.615927566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.615950813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.615990708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} {"msg":"Crucible 3eae879e-98b7-46fa-a65a-7ece1c721c40 has session id: 0c9dfb51-ac45-441b-9b59-9c8e607c680e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.616367312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Upstairs opts: Upstairs UUID: 3eae879e-98b7-46fa-a65a-7ece1c721c40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.616408667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Crucible stats registered with UUID: 3eae879e-98b7-46fa-a65a-7ece1c721c40","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.616428754Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:02.616442492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.616959428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617019251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} {"msg":"3eae879e-98b7-46fa-a65a-7ece1c721c40 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617081023Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617099191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617123038Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617137526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617151534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617164842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617178491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617191879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617211537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617228794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55441)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617255751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49880)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.61734411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53631)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.617592439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618033634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.61806655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618083088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618097846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618140321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618159639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618729708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618772553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.618805289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619388436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619416953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619431981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619444609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619458328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619471806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619484254Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619501772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[0]R flush_numbers[0..12]: [171, 168, 169, 169, 169, 168, 161, 171, 168, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.61951615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[0]R generation[0..12]: [105, 103, 104, 104, 104, 103, 99, 105, 103, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619530448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619544797Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[1]R flush_numbers[0..12]: [171, 168, 169, 169, 169, 168, 161, 171, 168, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619558775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[1]R generation[0..12]: [105, 103, 104, 104, 104, 103, 99, 105, 103, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619572463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619586672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[2]R flush_numbers[0..12]: [171, 168, 169, 169, 169, 168, 161, 171, 168, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.61960119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[2]R generation[0..12]: [105, 103, 104, 104, 104, 103, 99, 105, 103, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619615238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619629306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:45:02.619642615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619663222Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619676381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619689579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"":"downstairs","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619705037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619717565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"0","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619731104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"1","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619744382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"client":"2","":"downstairs","session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.61975778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619770279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"3eae879e-98b7-46fa-a65a-7ece1c721c40 is now active with session: 0c9dfb51-ac45-441b-9b59-9c8e607c680e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619783477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619795875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606,"session_id":"0c9dfb51-ac45-441b-9b59-9c8e607c680e"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:02.619809154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 337 len 40960 data: 7 7 9 9 7 7 7 7 9 7 02/30 Write block 423 len 24576 data: 11 11 11 10 9 12 03/30 Write block 136 len 28672 data: 7 6 6 6 5 6 4 04/30 Read block 542 len 40960 05/30 Write block 135 len 36864 data: 8 8 7 7 7 6 7 5 4 06/30 Flush 07/30 Write block 415 len 40960 data: 11 9 10 8 8 8 12 10 12 12 08/30 Write block 193 len 12288 data: 9 10 9 09/30 Write block 56 len 40960 data: 7 7 7 10 8 11 11 8 9 7 10/30 Read block 473 len 40960 11/30 Write block 112 len 24576 data: 8 9 10 10 10 8 12/30 Read block 588 len 12288 13/30 Read block 247 len 36864 14/30 Read block 27 len 4096 15/30 Write block 582 len 4096 data: 4 16/30 Read block 134 len 36864 17/30 Write block 18 len 12288 data: 15 16 15 18/30 Write block 83 len 28672 data: 9 10 11 12 9 10 11 19/30 Read block 186 len 24576 20/30 Read block 157 len 16384 21/30 Read block 288 len 4096 22/30 Read block 414 len 36864 23/30 Read block 73 len 28672 24/30 Read block 243 len 16384 25/30 Read block 432 len 12288 26/30 Read block 201 len 8192 27/30 Write block 287 len 16384 data: 8 12 10 7 28/30 Write block 327 len 20480 data: 11 9 8 6 5 29/30 Read block 197 len 12288 30/30 Write block 173 len 8192 data: 6 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 10 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:02.639536333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 171 171 171 T T T 1 020-039 103 103 103 168 168 168 T T T 2 040-059 104 104 104 169 169 169 T T T 3 060-079 104 104 104 169 169 169 T T T 4 080-099 104 104 104 169 169 169 T T T 5 100-119 103 103 103 168 168 168 T T T 6 120-139 106 106 106 172 172 172 F F F 7 140-159 106 106 106 172 172 172 F F F 8 160-179 103 103 103 168 168 168 T T T 9 180-199 105 105 105 171 171 171 T T T 10 200-219 105 105 105 171 171 171 F F F 11 220-239 105 105 105 171 171 171 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 104 104 104 170 170 170 T T T 15 300-319 104 104 104 169 169 169 F F F 16 320-339 106 106 106 172 172 172 T T T 17 340-359 106 106 106 172 172 172 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 105 105 105 171 171 171 T T T 21 420-439 106 106 106 172 172 172 T T T 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 105 105 105 171 171 171 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 105 105 105 171 171 171 T T T Max gen: 106, Max flush: 172 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 173 173 173 F F F 1 020-039 107 107 107 173 173 173 F F F 2 040-059 107 107 107 173 173 173 F F F 3 060-079 107 107 107 173 173 173 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 107 107 107 173 173 173 F F F 6 120-139 106 106 106 172 172 172 F F F 7 140-159 106 106 106 172 172 172 F F F 8 160-179 107 107 107 173 173 173 F F F 9 180-199 107 107 107 173 173 173 F F F 10 200-219 105 105 105 171 171 171 F F F 11 220-239 105 105 105 171 171 171 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 104 104 104 170 170 170 F F F 14 280-299 107 107 107 173 173 173 F F F 15 300-319 104 104 104 169 169 169 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 106 106 106 172 172 172 F F F 18 360-379 101 101 101 164 164 164 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 107 107 107 173 173 173 F F F 21 420-439 107 107 107 173 173 173 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 105 105 105 171 171 171 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 107, Max flush: 173 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.92151799Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.965048431Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.96569708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.965736175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.965750923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.965765962Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} {"msg":"Crucible 5e5002fe-5b63-4089-b123-db82f48a6caf has session id: 4a189925-8f34-4f44-bb07-1aa726c0fdd6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966175491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Upstairs opts: Upstairs UUID: 5e5002fe-5b63-4089-b123-db82f48a6caf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966220985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Crucible stats registered with UUID: 5e5002fe-5b63-4089-b123-db82f48a6caf","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966250472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:16.966273809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966812622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966837439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} {"msg":"5e5002fe-5b63-4089-b123-db82f48a6caf active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966909419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966928287Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966951004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966966123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966980521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.966994519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967008807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967022876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967044973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967072319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39058)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967178176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37414)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967216312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:16.967250747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:16.967284743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:16.967317389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.96738729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967931083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.9679564Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967971288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.967985356Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.968666681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.968693218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969231811Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969256248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969270366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969283115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969296933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:16.969314371Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.968072985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39459)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.968163544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.9694053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.969434406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.969981068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970519831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970544138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 173, 173, 173, 173, 172, 172, 173, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970561236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[0]R generation[0..12]: [107, 107, 107, 107, 107, 107, 106, 106, 107, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970576564Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970592412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 173, 173, 173, 173, 172, 172, 173, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.97060781Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[1]R generation[0..12]: [107, 107, 107, 107, 107, 107, 106, 106, 107, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970623078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970638236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 173, 173, 173, 173, 172, 172, 173, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970658274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[2]R generation[0..12]: [107, 107, 107, 107, 107, 107, 106, 106, 107, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970673462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.97068844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970702539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970716837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970730705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970744693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"":"downstairs","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970759061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.97077195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"0","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970786318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"1","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970800456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"client":"2","":"downstairs","session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970814714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970827973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"5e5002fe-5b63-4089-b123-db82f48a6caf is now active with session: 4a189925-8f34-4f44-bb07-1aa726c0fdd6","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:45:26.970841861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970856939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614,"session_id":"4a189925-8f34-4f44-bb07-1aa726c0fdd6"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:26.970870487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 8192 data: 7 10 02/30 Write block 272 len 4096 data: 4 03/30 Read block 360 len 40960 04/30 Read block 298 len 28672 05/30 Read block 73 len 16384 06/30 Read block 111 len 32768 07/30 Read block 65 len 24576 08/30 Read block 180 len 16384 09/30 Write block 496 len 40960 data: 13 11 9 11 11 9 10 12 11 10 10/30 Write block 364 len 4096 data: 12 11/30 Write block 135 len 20480 data: 9 9 8 8 8 12/30 Read block 390 len 16384 13/30 Flush 14/30 Write block 167 len 8192 data: 9 7 15/30 Write block 194 len 28672 data: 11 10 9 9 11 10 10 16/30 Write block 285 len 16384 data: 10 6 9 13 17/30 Write block 50 len 40960 data: 11 11 9 11 8 8 8 8 8 11 18/30 Write block 267 len 12288 data: 9 8 7 19/30 Write block 137 len 12288 data: 9 9 9 20/30 Flush 21/30 Write block 2 len 8192 data: 6 8 22/30 Write block 282 len 20480 data: 11 10 9 11 7 23/30 Read block 281 len 4096 24/30 Flush 25/30 Write block 484 len 28672 data: 7 7 7 7 7 8 8 26/30 Write block 111 len 36864 data: 8 9 10 11 11 11 9 8 8 27/30 Write block 481 len 12288 data: 13 11 11 28/30 Write block 17 len 12288 data: 13 16 17 29/30 Read block 31 len 16384 30/30 Write block 29 len 8192 data: 11 11 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 1 1 5 Done 3 5 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:26.988486818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 176 176 176 T T T 1 020-039 107 107 107 173 173 173 T T T 2 040-059 108 108 108 175 175 175 F F F 3 060-079 107 107 107 173 173 173 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 107 107 107 173 173 173 T T T 6 120-139 108 108 108 175 175 175 F F F 7 140-159 106 106 106 172 172 172 F F F 8 160-179 108 108 108 175 175 175 F F F 9 180-199 108 108 108 175 175 175 F F F 10 200-219 108 108 108 175 175 175 F F F 11 220-239 105 105 105 171 171 171 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 108 108 108 175 175 175 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 104 104 104 169 169 169 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 106 106 106 172 172 172 F F F 18 360-379 108 108 108 174 174 174 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 107 107 107 173 173 173 F F F 21 420-439 107 107 107 173 173 173 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 108 108 108 174 174 174 T T T 25 500-519 108 108 108 174 174 174 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 108, Max flush: 176 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 177 177 177 F F F 1 020-039 109 109 109 177 177 177 F F F 2 040-059 108 108 108 175 175 175 F F F 3 060-079 107 107 107 173 173 173 F F F 4 080-099 107 107 107 173 173 173 F F F 5 100-119 109 109 109 177 177 177 F F F 6 120-139 108 108 108 175 175 175 F F F 7 140-159 106 106 106 172 172 172 F F F 8 160-179 108 108 108 175 175 175 F F F 9 180-199 108 108 108 175 175 175 F F F 10 200-219 108 108 108 175 175 175 F F F 11 220-239 105 105 105 171 171 171 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 108 108 108 175 175 175 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 104 104 104 169 169 169 F F F 16 320-339 107 107 107 173 173 173 F F F 17 340-359 106 106 106 172 172 172 F F F 18 360-379 108 108 108 174 174 174 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 107 107 107 173 173 173 F F F 21 420-439 107 107 107 173 173 173 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 109 109 109 177 177 177 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 99 99 99 161 161 161 F F F 27 540-559 102 102 102 166 166 166 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 109, Max flush: 177 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.2332645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:31.277828522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.278529585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.310966364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312072186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312114721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312135378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312150607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} {"msg":"Crucible 224cc4b0-1552-4cd9-9471-134648dd0219 has session id: 44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31260867Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Upstairs opts: Upstairs UUID: 224cc4b0-1552-4cd9-9471-134648dd0219, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312650924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Crucible stats registered with UUID: 224cc4b0-1552-4cd9-9471-134648dd0219","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.312675951Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:31.312698048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313239541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313276057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} {"msg":"224cc4b0-1552-4cd9-9471-134648dd0219 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313333649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313362016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313389323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31341483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313442646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313468503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313495889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313537054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31356972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313600187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48862)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313630543Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64339)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313662239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65419)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.313695065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.314573286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.314604082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31462086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.314635728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.314650426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.314665204Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315177071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315313294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31534321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315751079Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315791544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315816621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315884432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.3159048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.315918808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31598917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316033414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[0]R flush_numbers[0..12]: [177, 177, 175, 173, 173, 177, 175, 172, 175, 175, 175, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316049392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[0]R generation[0..12]: [109, 109, 108, 107, 107, 109, 108, 106, 108, 108, 108, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31606381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316078139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[1]R flush_numbers[0..12]: [177, 177, 175, 173, 173, 177, 175, 172, 175, 175, 175, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316092857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[1]R generation[0..12]: [109, 109, 108, 107, 107, 109, 108, 106, 108, 108, 108, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316107165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316121523Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[2]R flush_numbers[0..12]: [177, 177, 175, 173, 173, 177, 175, 172, 175, 175, 175, 171]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316135421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[2]R generation[0..12]: [109, 109, 108, 107, 107, 109, 108, 106, 108, 108, 108, 105]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316153319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316167787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316181066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316194394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316207612Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316220721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"":"downstairs","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316234179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316246507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622Wait for a query_work_queue command to finish before sending IO ,"client":"0","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316328088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"1","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316344745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"client":"2","":"downstairs","session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316358284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316370882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"224cc4b0-1552-4cd9-9471-134648dd0219 is now active with session: 44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.31638419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316396649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622,"session_id":"44269e9e-2d1d-4ba9-a4a0-633fc31c6ef4"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:31.316409087Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 16384 data: 7 7 7 7 02/30 Read block 369 len 28672 03/30 Read block 517 len 20480 04/30 Write block 317 len 16384 data: 9 9 10 12 05/30 Write block 60 len 12288 data: 9 12 12 06/30 Write block 530 len 20480 data: 5 8 9 10 9 07/30 Write block 105 len 28672 data: 9 9 6 5 5 6 9 08/30 Write block 88 len 12288 data: 11 12 8 09/30 Write block 558 len 4096 data: 9 10/30 Read block 313 len 12288 11/30 Read block 543 len 24576 12/30 Write block 354 len 28672 data: 6 9 9 6 7 9 9 13/30 Read block 17 len 28672 14/30 Read block 483 len 32768 15/30 Read block 64 len 8192 16/30 Write block 425 len 20480 data: 12 11 10 13 12 17/30 Flush 18/30 Write block 151 len 32768 data: 10 14 15 15 14 15 14 10 19/30 Write block 112 len 16384 data: 10 11 12 12 20/30 Write block 232 len 8192 data: 8 7 21/30 Write block 417 len 16384 data: 11 9 9 9 22/30 Read block 136 len 32768 23/30 Read block 45 len 16384 24/30 Read block 568 len 32768 25/30 Read block 512 len 4096 26/30 Read block 73 len 28672 27/30 Write block 415 len 20480 data: 12 10 12 10 10 28/30 Read block 239 len 32768 29/30 Read block 431 len 28672 30/30 Read block 586 len 32768 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:31.336433889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 177 177 177 F F F 1 020-039 109 109 109 177 177 177 F F F 2 040-059 108 108 108 175 175 175 F F F 3 060-079 110 110 110 178 178 178 F F F 4 080-099 110 110 110 178 178 178 F F F 5 100-119 110 110 110 178 178 178 T T T 6 120-139 108 108 108 175 175 175 F F F 7 140-159 106 106 106 172 172 172 T T T 8 160-179 108 108 108 175 175 175 F F F 9 180-199 108 108 108 175 175 175 F F F 10 200-219 110 110 110 178 178 178 F F F 11 220-239 105 105 105 171 171 171 T T T 12 240-259 105 105 105 171 171 171 F F F 13 260-279 108 108 108 175 175 175 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 110 110 110 178 178 178 F F F 16 320-339 110 110 110 178 178 178 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 110 110 110 178 178 178 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 107 107 107 173 173 173 T T T 21 420-439 110 110 110 178 178 178 T T T 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 109 109 109 177 177 177 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 110, Max flush: 178 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 177 177 177 F F F 1 020-039 109 109 109 177 177 177 F F F 2 040-059 108 108 108 175 175 175 F F F 3 060-079 110 110 110 178 178 178 F F F 4 080-099 110 110 110 178 178 178 F F F 5 100-119 111 111 111 179 179 179 F F F 6 120-139 108 108 108 175 175 175 F F F 7 140-159 111 111 111 179 179 179 F F F 8 160-179 108 108 108 175 175 175 F F F 9 180-199 108 108 108 175 175 175 F F F 10 200-219 110 110 110 178 178 178 F F F 11 220-239 111 111 111 179 179 179 F F F 12 240-259 105 105 105 171 171 171 F F F 13 260-279 108 108 108 175 175 175 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 110 110 110 178 178 178 F F F 16 320-339 110 110 110 178 178 178 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 110 110 110 178 178 178 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 111 111 111 179 179 179 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 101 101 101 164 164 164 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 109 109 109 177 177 177 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 111, Max flush: 179 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.61396448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.657362997Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658335366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658370362Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.65838486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658400368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} {"msg":"Crucible 28543561-add6-4669-b242-a27579ac5e1a has session id: ce0f44b2-5797-4555-ae67-011a06bc6966","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658921503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Upstairs opts: Upstairs UUID: 28543561-add6-4669-b242-a27579ac5e1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658964638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Crucible stats registered with UUID: 28543561-add6-4669-b242-a27579ac5e1a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.658989155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:35.659011462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659521369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659549105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} {"msg":"28543561-add6-4669-b242-a27579ac5e1a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659638245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659658302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659682579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659699367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659724354Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.65975116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659779617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659807004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.65983508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.659957505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_connection connected from Ok(127.0.0.1:34058)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660057013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_connection connected from Ok(127.0.0.1:39902)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660088849Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:35.660115345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:35.660141242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:35.660168089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660188946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.66056143Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660592216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660608454Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.660626432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661223768Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661373489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661707267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661736724Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661752282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661932729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661953227Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:35.661967155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.662013949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_connection connected from Ok(127.0.0.1:47714)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.662081381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.663313178Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.663344044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.663831093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664374696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664399193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[0]R flush_numbers[0..12]: [177, 177, 175, 178, 178, 179, 175, 179, 175, 175, 178, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664415421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[0]R generation[0..12]: [109, 109, 108, 110, 110, 111, 108, 111, 108, 108, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664430819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664445967Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[1]R flush_numbers[0..12]: [177, 177, 175, 178, 178, 179, 175, 179, 175, 175, 178, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664466674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[1]R generation[0..12]: [109, 109, 108, 110, 110, 111, 108, 111, 108, 108, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664481973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664497001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[2]R flush_numbers[0..12]: [177, 177, 175, 178, 178, 179, 175, 179, 175, 175, 178, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664511779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[2]R generation[0..12]: [109, 109, 108, 110, 110, 111, 108, 111, 108, 108, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664526557Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664541655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664555494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664569652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.66458344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664597318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"":"downstairs","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664611387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664624195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"0","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664638413Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"1","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664652591Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"client":"2","":"downstairs","session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664671709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664687817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"28543561-add6-4669-b242-a27579ac5e1a is now active with session: ce0f44b2-5797-4555-ae67-011a06bc6966","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664701755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664714504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630,"session_id":"ce0f44b2-5797-4555-ae67-011a06bc6966"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:45.664727582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 40960 data: 7 7 7 9 6 6 8 10 10 11 02/30 Read block 143 len 20480 03/30 Flush 04/30 Flush 05/30 Write block 23 len 28672 data: 17 14 15 13 14 11 12 06/30 Write block 115 len 40960 data: 13 12 10 9 9 7 8 9 7 7 07/30 Write block 181 len 36864 data: 12 10 11 11 9 9 7 8 6 08/30 Write block 247 len 16384 data: 5 6 7 9 09/30 Write block 56 len 16384 data: 9 9 9 12 10/30 Write block 377 len 8192 data: 10 9 11/30 Write block 258 len 20480 data: 6 5 6 8 9 12/30 Write block 200 len 12288 data: 11 10 10 13/30 Write block 1 len 16384 data: 5 7 9 8 14/30 Write block 151 len 20480 data: 11 15 16 16 15 15/30 Write block 444 len 40960 data: 11 9 9 11 6 6 8 7 9 10 16/30 Write block 322 len 28672 data: 12 12 10 10 11 12 10 17/30 Write block 108 len 28672 data: 6 6 7 10 11 12 13 18/30 Read block 562 len 12288 19/30 Read block 79 len 40960 20/30 Flush 21/30 Write block 452 len 16384 data: 10 11 10 8 22/30 Write block 73 len 12288 data: 8 8 7 23/30 Read block 110 len 28672 24/30 Read block 37 len 36864 25/30 Read block 239 len 36864 26/30 Read block 164 len 12288 27/30 Read block 594 len 20480 28/30 Read block 352 len 16384 29/30 Read block 412 len 36864 30/30 Write block 433 len 40960 data: 9 11 14 12 10 12 14 12 9 11 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:45.684080797Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 182 182 182 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 112 112 112 182 182 182 F F F 3 060-079 110 110 110 178 178 178 T T T 4 080-099 110 110 110 178 178 178 F F F 5 100-119 112 112 112 182 182 182 F F F 6 120-139 112 112 112 182 182 182 F F F 7 140-159 112 112 112 182 182 182 F F F 8 160-179 108 108 108 175 175 175 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 112 112 112 182 182 182 F F F 11 220-239 111 111 111 179 179 179 F F F 12 240-259 112 112 112 182 182 182 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 112 112 112 180 180 180 F F F 16 320-339 112 112 112 182 182 182 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 111 111 111 179 179 179 F F F 21 420-439 111 111 111 179 179 179 T T T 22 440-459 112 112 112 182 182 182 T T T 23 460-479 105 105 105 171 171 171 F F F 24 480-499 109 109 109 177 177 177 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 112, Max flush: 182 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 182 182 182 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 112 112 112 182 182 182 F F F 3 060-079 113 113 113 183 183 183 F F F 4 080-099 110 110 110 178 178 178 F F F 5 100-119 112 112 112 182 182 182 F F F 6 120-139 112 112 112 182 182 182 F F F 7 140-159 112 112 112 182 182 182 F F F 8 160-179 108 108 108 175 175 175 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 112 112 112 182 182 182 F F F 11 220-239 111 111 111 179 179 179 F F F 12 240-259 112 112 112 182 182 182 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 112 112 112 180 180 180 F F F 16 320-339 112 112 112 182 182 182 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 103 103 103 168 168 168 F F F 20 400-419 111 111 111 179 179 179 F F F 21 420-439 113 113 113 183 183 183 F F F 22 440-459 113 113 113 183 183 183 F F F 23 460-479 105 105 105 171 171 171 F F F 24 480-499 109 109 109 177 177 177 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 113, Max flush: 183 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.913505106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:49.956406275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.956808385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.988140061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.988746936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.988783261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.988797709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.988812408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} {"msg":"Crucible 66decacc-ce15-43f7-b77d-c182bb8104c3 has session id: 283070af-2752-4bce-92bd-9240b958f70b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989314155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Upstairs opts: Upstairs UUID: 66decacc-ce15-43f7-b77d-c182bb8104c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.98935452Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Crucible stats registered with UUID: 66decacc-ce15-43f7-b77d-c182bb8104c3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989376857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:49.989391086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.98984138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989877305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} {"msg":"66decacc-ce15-43f7-b77d-c182bb8104c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989892453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989906002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.989975813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990015858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990045854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990072751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990099708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990126224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990155331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990183487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51017)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990306822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44443)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990344757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43278)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990371584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990835166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990863263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990878811Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.990893509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.991006095Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.991052489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.991497394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.991561796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.991824903Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992063674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992088301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992102599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992160832Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992179989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992193707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992380394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992409381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[0]R flush_numbers[0..12]: [182, 182, 182, 183, 178, 182, 182, 182, 175, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992425419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[0]R generation[0..12]: [112, 112, 112, 113, 110, 112, 112, 112, 108, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992440237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992454925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[1]R flush_numbers[0..12]: [182, 182, 182, 183, 178, 182, 182, 182, 175, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992473413Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[1]R generation[0..12]: [112, 112, 112, 113, 110, 112, 112, 112, 108, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992488221Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992502469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[2]R flush_numbers[0..12]: [182, 182, 182, 183, 178, 182, 182, 182, 175, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992523187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[2]R generation[0..12]: [112, 112, 112, 113, 110, 112, 112, 112, 108, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992550533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992604607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992619385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992633423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:45:49.99265729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"":"downstairs","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992688406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992712923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"0","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992729071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"1","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992742599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"client":"2","":"downstairs","session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992764567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992788624Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"66decacc-ce15-43f7-b77d-c182bb8104c3 is now active with session: 283070af-2752-4bce-92bd-9240b958f70b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.99281552Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992834758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638,"session_id":"283070af-2752-4bce-92bd-9240b958f70b"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:49.992847986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 28672 data: 8 10 9 10 13 15 16 02/30 Write block 337 len 12288 data: 8 8 10 03/30 Read block 234 len 20480 04/30 Read block 314 len 40960 05/30 Write block 175 len 20480 data: 6 12 11 13 12 06/30 Flush 07/30 Write block 308 len 24576 data: 6 7 8 8 8 10 08/30 Flush 09/30 Write block 484 len 20480 data: 8 8 8 8 8 10/30 Write block 397 len 24576 data: 13 13 14 12 12 12 11/30 Read block 207 len 28672 12/30 Write block 211 len 40960 data: 13 12 12 9 9 10 7 6 4 6 13/30 Write block 459 len 36864 data: 8 9 8 8 6 7 7 8 7 14/30 Write block 86 len 20480 data: 13 10 12 13 9 15/30 Read block 484 len 40960 16/30 Write block 474 len 40960 data: 8 11 11 12 13 13 13 14 12 12 17/30 Write block 96 len 16384 data: 3 5 5 5 18/30 Write block 64 len 20480 data: 10 8 6 6 8 19/30 Write block 216 len 12288 data: 11 8 7 20/30 Write block 234 len 28672 data: 8 8 8 6 7 7 7 21/30 Read block 123 len 20480 22/30 Write block 97 len 24576 data: 6 6 6 7 9 9 23/30 Read block 131 len 36864 24/30 Read block 375 len 20480 25/30 Read block 96 len 32768 26/30 Read block 2 len 12288 27/30 Write block 422 len 20480 data: 11 13 13 13 12 28/30 Write block 247 len 28672 data: 6 7 8 10 6 6 8 29/30 Read block 98 len 28672 30/30 Write block 143 len 16384 data: 5 5 8 8 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:50.013215525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 184 184 184 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 112 112 112 182 182 182 F F F 3 060-079 113 113 113 183 183 183 T T T 4 080-099 110 110 110 178 178 178 T T T 5 100-119 112 112 112 182 182 182 T T T 6 120-139 112 112 112 182 182 182 F F F 7 140-159 112 112 112 182 182 182 T T T 8 160-179 114 114 114 184 184 184 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 112 112 112 182 182 182 T T T 11 220-239 111 111 111 179 179 179 T T T 12 240-259 112 112 112 182 182 182 T T T 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 114 114 114 184 184 184 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 103 103 103 168 168 168 T T T 20 400-419 111 111 111 179 179 179 T T T 21 420-439 113 113 113 183 183 183 T T T 22 440-459 113 113 113 183 183 183 T T T 23 460-479 105 105 105 171 171 171 T T T 24 480-499 109 109 109 177 177 177 T T T 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 114, Max flush: 185 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 184 184 184 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 112 112 112 182 182 182 F F F 3 060-079 115 115 115 186 186 186 F F F 4 080-099 115 115 115 186 186 186 F F F 5 100-119 115 115 115 186 186 186 F F F 6 120-139 112 112 112 182 182 182 F F F 7 140-159 115 115 115 186 186 186 F F F 8 160-179 114 114 114 184 184 184 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 115 115 115 186 186 186 F F F 11 220-239 115 115 115 186 186 186 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 114 114 114 184 184 184 F F F 17 340-359 110 110 110 178 178 178 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 115 115 115 186 186 186 F F F 20 400-419 115 115 115 186 186 186 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 115 115 115 186 186 186 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 115, Max flush: 186 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.330776242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:54.373655814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.37424605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.406058147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.406966924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407013678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407033626Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407049594Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} {"msg":"Crucible 3e74ca69-135f-4cd9-93b5-b2e21adf8f51 has session id: 00ca5473-aa0a-4da9-b6fd-3bac92690b34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407501948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Upstairs opts: Upstairs UUID: 3e74ca69-135f-4cd9-93b5-b2e21adf8f51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407546342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Crucible stats registered with UUID: 3e74ca69-135f-4cd9-93b5-b2e21adf8f51","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.407568389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:54.407583058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408076046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408140878Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} {"msg":"3e74ca69-135f-4cd9-93b5-b2e21adf8f51 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408157036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408170905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408197111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408213699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408228218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408242756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408257024Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408271132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.40828583Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408300458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:65273)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408438631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:54981)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408669743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:51797)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.408711337Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409266398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409342929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409372305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409395432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409423839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.40949223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409926296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.409969641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410014925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410581375Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.41062216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410640098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410653326Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410667584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410681183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410693471Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.41070679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[0]R flush_numbers[0..12]: [184, 182, 182, 186, 186, 186, 182, 186, 184, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410725837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[0]R generation[0..12]: [114, 112, 112, 115, 115, 115, 112, 115, 114, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410740495Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410755333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[1]R flush_numbers[0..12]: [184, 182, 182, 186, 186, 186, 182, 186, 184, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410769222Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[1]R generation[0..12]: [114, 112, 112, 115, 115, 115, 112, 115, 114, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.41078311Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410797668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[2]R flush_numbers[0..12]: [184, 182, 182, 186, 186, 186, 182, 186, 184, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410811786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[2]R generation[0..12]: [114, 112, 112, 115, 115, 115, 112, 115, 114, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410825565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410839333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.41085896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410882957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410905855Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410920493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"":"downstairs","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410933801Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410950089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"0","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410974516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"1","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.410992984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"client":"2","":"downstairs","session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.411007622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.411020011Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"3e74ca69-135f-4cd9-93b5-b2e21adf8f51 is now active with session: 00ca5473-aa0a-4da9-b6fd-3bac92690b34","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.411032819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.411050307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646,"session_id":"00ca5473-aa0a-4da9-b6fd-3bac92690b34"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:54.411073374Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 59 len 40960 data: 13 10 13 13 9 11 9 7 7 9 02/30 Read block 251 len 36864 03/30 Read block 470 len 12288 04/30 Write block 380 len 20480 data: 7 7 6 6 5 05/30 Read block 490 len 40960 06/30 Write block 92 len 20480 data: 5 3 3 2 4 07/30 Read block 395 len 16384 08/30 Write block 227 len 36864 data: 10 10 11 9 10 9 8 9 9 09/30 Write block 155 len 12288 data: 16 16 15 10/30 Write block 171 len 12288 data: 5 5 7 11/30 Read block 136 len 12288 12/30 Read block 76 len 16384 13/30 Read block 452 len 32768 14/30 Write block 352 len 8192 data: 7 7 15/30 Read block 239 len 20480 16/30 Write block 232 len 20480 data: 10 9 10 10 9 17/30 Flush 18/30 Write block 487 len 16384 data: 9 9 9 9 19/30 Read block 282 len 24576 20/30 Read block 159 len 20480 21/30 Write block 495 len 20480 data: 13 14 12 10 12 22/30 Read block 570 len 28672 23/30 Write block 164 len 36864 data: 11 10 9 10 8 7 6 6 6 24/30 Flush 25/30 Write block 72 len 16384 data: 9 9 9 8 26/30 Write block 205 len 8192 data: 8 8 27/30 Read block 20 len 36864 28/30 Read block 165 len 20480 29/30 Read block 191 len 36864 30/30 Write block 65 len 36864 data: 10 8 8 10 7 9 11 10 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:54.432611937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 184 184 184 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 116 116 116 187 187 187 T T T 4 080-099 116 116 116 187 187 187 F F F 5 100-119 115 115 115 186 186 186 F F F 6 120-139 112 112 112 182 182 182 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 116 116 116 188 188 188 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 115 115 115 186 186 186 T T T 11 220-239 116 116 116 187 187 187 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 114 114 114 184 184 184 F F F 17 340-359 116 116 116 187 187 187 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 116 116 116 187 187 187 F F F 20 400-419 115 115 115 186 186 186 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 116, Max flush: 188 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 184 184 184 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 117 117 117 189 189 189 F F F 4 080-099 116 116 116 187 187 187 F F F 5 100-119 115 115 115 186 186 186 F F F 6 120-139 112 112 112 182 182 182 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 116 116 116 188 188 188 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 117 117 117 189 189 189 F F F 11 220-239 116 116 116 187 187 187 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 114 114 114 184 184 184 F F F 17 340-359 116 116 116 187 187 187 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 116 116 116 187 187 187 F F F 20 400-419 115 115 115 186 186 186 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 F F F 24 480-499 116 116 116 188 188 188 F F F 25 500-519 108 108 108 174 174 174 F F F 26 520-539 110 110 110 178 178 178 F F F 27 540-559 110 110 110 178 178 178 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 117, Max flush: 189 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.698896766Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:58.741492252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.741952965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.773313468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.773987244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774025939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774040667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774056365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} {"msg":"Crucible c91d07ca-d2d5-4c10-9041-a08972b66e5d has session id: 7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774494921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Upstairs opts: Upstairs UUID: c91d07ca-d2d5-4c10-9041-a08972b66e5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774543404Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Crucible stats registered with UUID: c91d07ca-d2d5-4c10-9041-a08972b66e5d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774569111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:58.774587679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.774996628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775039643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} {"msg":"c91d07ca-d2d5-4c10-9041-a08972b66e5d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775095636Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775128082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775166807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775194994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.77522387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775250667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775278883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.77530541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775331237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775359513Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48681)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775391239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41310)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775423045Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43528)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.775528902Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776056487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776085843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.77610584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776120848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776199149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776220436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776823351Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776866336Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.776901271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777434785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777463762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.77747894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777551181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777571848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777586706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777599075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777612643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[0]R flush_numbers[0..12]: [184, 182, 187, 189, 187, 186, 182, 187, 188, 182, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777626541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[0]R generation[0..12]: [114, 112, 116, 117, 116, 115, 112, 116, 116, 112, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777644899Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777659677Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[1]R flush_numbers[0..12]: [184, 182, 187, 189, 187, 186, 182, 187, 188, 182, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777673795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[1]R generation[0..12]: [114, 112, 116, 117, 116, 115, 112, 116, 116, 112, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777687684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777701892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[2]R flush_numbers[0..12]: [184, 182, 187, 189, 187, 186, 182, 187, 188, 182, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.77771586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[2]R generation[0..12]: [114, 112, 116, 117, 116, 115, 112, 116, 116, 112, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777729679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777743747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777756905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777770154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777783352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.7777965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777816448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777828866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"0","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777844824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"1","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777858863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"client":"2","":"downstairs","session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777872601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777884979Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"c91d07ca-d2d5-4c10-9041-a08972b66e5d is now active with session: 7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777897718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777909836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654,"session_id":"7fc5571d-cf2e-4ee2-bc9b-29f6b8834ed8"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:45:58.777922135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 75 len 4096 data: 9 02/30 Read block 70 len 12288 03/30 Write block 321 len 32768 data: 11 13 13 11 11 12 13 11 04/30 Write block 127 len 4096 data: 9 05/30 Write block 409 len 36864 data: 9 8 8 8 11 11 13 11 13 06/30 Read block 9 len 20480 07/30 Write block 512 len 20480 data: 12 12 9 9 8 08/30 Flush 09/30 Write block 92 len 20480 data: 6 4 4 3 5 10/30 Write block 111 len 28672 data: 11 12 13 14 14 13 11 11/30 Read block 131 len 28672 12/30 Read block 52 len 24576 13/30 Write block 388 len 16384 data: 7 7 8 9 14/30 Write block 166 len 36864 data: 10 11 9 8 7 7 7 8 9 15/30 Write block 285 len 36864 data: 12 8 10 14 11 8 7 7 8 16/30 Read block 163 len 4096 17/30 Write block 537 len 40960 data: 9 9 11 8 7 8 9 9 9 6 18/30 Read block 17 len 12288 19/30 Write block 9 len 4096 data: 15 20/30 Write block 333 len 36864 data: 5 5 6 7 9 9 11 10 8 21/30 Read block 392 len 36864 22/30 Write block 223 len 36864 data: 8 8 9 11 11 11 12 10 11 23/30 Write block 476 len 40960 data: 12 13 14 14 14 15 13 13 9 9 24/30 Read block 491 len 32768 25/30 Read block 203 len 8192 26/30 Write block 217 len 8192 data: 9 8 27/30 Read block 477 len 12288 28/30 Write block 328 len 20480 data: 12 9 7 6 5 29/30 Read block 564 len 40960 30/30 Write block 64 len 20480 data: 12 11 9 9 11 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:45:58.797851048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 184 184 184 T T T 1 020-039 112 112 112 182 182 182 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 118 118 118 190 190 190 T T T 4 080-099 116 116 116 187 187 187 T T T 5 100-119 115 115 115 186 186 186 T T T 6 120-139 118 118 118 190 190 190 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 116 116 116 188 188 188 T T T 9 180-199 112 112 112 182 182 182 F F F 10 200-219 117 117 117 189 189 189 T T T 11 220-239 116 116 116 187 187 187 T T T 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 108 108 108 176 176 176 T T T 15 300-319 114 114 114 185 185 185 F F F 16 320-339 118 118 118 190 190 190 T T T 17 340-359 116 116 116 187 187 187 T T T 18 360-379 112 112 112 182 182 182 F F F 19 380-399 116 116 116 187 187 187 T T T 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 115 115 115 186 186 186 T T T 24 480-499 116 116 116 188 188 188 T T T 25 500-519 118 118 118 190 190 190 F F F 26 520-539 110 110 110 178 178 178 T T T 27 540-559 110 110 110 178 178 178 T T T 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 118, Max flush: 190 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 112 112 112 182 182 182 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 119 119 119 191 191 191 F F F 6 120-139 118 118 118 190 190 190 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 119 119 119 191 191 191 F F F 9 180-199 112 112 112 182 182 182 F F F 10 200-219 119 119 119 191 191 191 F F F 11 220-239 119 119 119 191 191 191 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 F F F 14 280-299 119 119 119 191 191 191 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 119 119 119 191 191 191 F F F 17 340-359 119 119 119 191 191 191 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 119 119 119 191 191 191 F F F 24 480-499 119 119 119 191 191 191 F F F 25 500-519 118 118 118 190 190 190 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 119 119 119 191 191 191 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 107 107 107 173 173 173 F F F Max gen: 119, Max flush: 191 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.125429359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:03.167144056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.167531167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.198215724Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.19881704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.198852245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.198866473Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.198881662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} {"msg":"Crucible 42ec3a6e-b4ca-4268-93c7-5e3b64014e9d has session id: a6e358fc-862e-4db5-aba2-70157d664adf","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199359662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Upstairs opts: Upstairs UUID: 42ec3a6e-b4ca-4268-93c7-5e3b64014e9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199391908Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Crucible stats registered with UUID: 42ec3a6e-b4ca-4268-93c7-5e3b64014e9d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199413176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:03.199427854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199796348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199820795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} {"msg":"42ec3a6e-b4ca-4268-93c7-5e3b64014e9d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199871948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199900645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.19993886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199955668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199974346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.199998613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20002329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200039368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200054226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200068094Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43991)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200157503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:55709)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200194119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_connection connected from Ok(127.0.0.1:33616)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200225265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20082388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200861386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200919788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.200950565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.201018146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.201042863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.201484338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.201526503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20162926Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202095152Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202123369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202138347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202151195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202165084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202182821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.2021957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202209188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[0]R flush_numbers[0..12]: [191, 182, 187, 191, 191, 191, 190, 187, 191, 182, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202223266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[0]R generation[0..12]: [119, 112, 116, 119, 119, 119, 118, 116, 119, 112, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202237265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202252553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[1]R flush_numbers[0..12]: [191, 182, 187, 191, 191, 191, 190, 187, 191, 182, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20227663Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[1]R generation[0..12]: [119, 112, 116, 119, 119, 119, 118, 116, 119, 112, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202294918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202309406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[2]R flush_numbers[0..12]: [191, 182, 187, 191, 191, 191, 190, 187, 191, 182, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202324344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[2]R generation[0..12]: [119, 112, 116, 119, 119, 119, 118, 116, 119, 112, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202338242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20235241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:46:03.202365459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202382447Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202398325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202411703Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"":"downstairs","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202425131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.20243725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"0","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202450398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"1","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202463837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"client":"2","":"downstairs","session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202477515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202489943Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"42ec3a6e-b4ca-4268-93c7-5e3b64014e9d is now active with session: a6e358fc-862e-4db5-aba2-70157d664adf","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202502812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662,"session_id":"a6e358fc-862e-4db5-aba2-70157d664adf"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:03.202527359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 16384 data: 9 7 6 7 02/30 Flush 03/30 Write block 28 len 28672 data: 12 13 12 11 11 11 12 04/30 Flush 05/30 Flush 06/30 Write block 220 len 40960 data: 7 7 8 9 9 10 12 12 12 13 07/30 Flush 08/30 Write block 172 len 36864 data: 8 9 10 7 13 12 14 13 13 09/30 Read block 350 len 20480 10/30 Flush 11/30 Flush 12/30 Write block 545 len 12288 data: 10 7 7 13/30 Write block 160 len 32768 data: 7 9 7 8 12 11 11 12 14/30 Read block 592 len 8192 15/30 Write block 590 len 12288 data: 8 9 8 16/30 Flush 17/30 Write block 291 len 4096 data: 8 18/30 Flush 19/30 Flush 20/30 Write block 326 len 4096 data: 13 21/30 Write block 198 len 32768 data: 12 11 12 11 11 8 8 9 22/30 Write block 192 len 20480 data: 7 10 12 11 10 23/30 Read block 585 len 16384 24/30 Read block 85 len 28672 25/30 Read block 329 len 32768 26/30 Read block 133 len 12288 27/30 Write block 291 len 8192 data: 9 8 28/30 Read block 41 len 8192 29/30 Read block 294 len 36864 30/30 Write block 276 len 12288 data: 8 7 8 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:03.221156894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 120 120 120 193 193 193 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 119 119 119 191 191 191 F F F 6 120-139 118 118 118 190 190 190 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 120 120 120 198 198 198 F F F 9 180-199 120 120 120 196 196 196 T T T 10 200-219 119 119 119 191 191 191 T T T 11 220-239 120 120 120 195 195 195 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 112 112 112 182 182 182 T T T 14 280-299 120 120 120 199 199 199 T T T 15 300-319 114 114 114 185 185 185 F F F 16 320-339 119 119 119 191 191 191 T T T 17 340-359 119 119 119 191 191 191 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 119 119 119 191 191 191 F F F 24 480-499 119 119 119 191 191 191 F F F 25 500-519 118 118 118 190 190 190 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 120 120 120 198 198 198 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 120 120 120 198 198 198 F F F Max gen: 120, Max flush: 199 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 120 120 120 193 193 193 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 119 119 119 191 191 191 F F F 6 120-139 118 118 118 190 190 190 F F F 7 140-159 116 116 116 187 187 187 F F F 8 160-179 120 120 120 198 198 198 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 121 121 121 200 200 200 F F F 11 220-239 120 120 120 195 195 195 F F F 12 240-259 115 115 115 186 186 186 F F F 13 260-279 121 121 121 200 200 200 F F F 14 280-299 121 121 121 200 200 200 F F F 15 300-319 114 114 114 185 185 185 F F F 16 320-339 121 121 121 200 200 200 F F F 17 340-359 119 119 119 191 191 191 F F F 18 360-379 112 112 112 182 182 182 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 119 119 119 191 191 191 F F F 24 480-499 119 119 119 191 191 191 F F F 25 500-519 118 118 118 190 190 190 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 120 120 120 198 198 198 F F F 28 560-579 101 101 101 164 164 164 F F F 29 580-599 120 120 120 198 198 198 F F F Max gen: 121, Max flush: 200 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.490403604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.534271163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.534878657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.534915403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.534931161Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.534946749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} {"msg":"Crucible 9ae0f70b-1807-493e-9bf0-8cf7cc4e5247 has session id: b746a378-e6d6-4c2f-9304-ce4c865fd82e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53542514Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Upstairs opts: Upstairs UUID: 9ae0f70b-1807-493e-9bf0-8cf7cc4e5247, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.535467494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Crucible stats registered with UUID: 9ae0f70b-1807-493e-9bf0-8cf7cc4e5247","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.535492511Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:07.535514458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536088897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536115704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} {"msg":"9ae0f70b-1807-493e-9bf0-8cf7cc4e5247 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536156009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536172217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536194064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536208572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53622342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536237219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536251047Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536264575Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536327657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536348555Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39716)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536375191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44039)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536402188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35940)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.536476719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537064716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537097582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53711481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537129828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537288058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537329653Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537726464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53775605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.537844509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53831864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538358915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538389921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538413458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538437185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538461322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538483489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538508497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[0]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 191, 190, 187, 198, 200, 200, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538537993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[0]R generation[0..12]: [119, 120, 116, 119, 119, 119, 118, 116, 120, 121, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.53856298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538589686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[1]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 191, 190, 187, 198, 200, 200, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538617843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[1]R generation[0..12]: [119, 120, 116, 119, 119, 119, 118, 116, 120, 121, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538644369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538673506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[2]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 191, 190, 187, 198, 200, 200, 195]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538704752Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[2]R generation[0..12]: [119, 120, 116, 119, 119, 119, 118, 116, 120, 121, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538744737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538775953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538805189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538835076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538862982Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538892019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"":"downstairs","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538924995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538956171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"0","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538980288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"1","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.538994786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"client":"2","":"downstairs","session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.539008544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.539021343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"9ae0f70b-1807-493e-9bf0-8cf7cc4e5247 is now active with session: b746a378-e6d6-4c2f-9304-ce4c865fd82e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.539034441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.539046759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670,"session_id":"b746a378-e6d6-4c2f-9304-ce4c865fd82e"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:07.539059298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 588 len 40960 data: 4 7 9 10 9 9 10 8 5 5 02/30 Read block 515 len 12288 03/30 Read block 38 len 20480 04/30 Write block 371 len 20480 data: 11 10 8 6 6 05/30 Read block 453 len 32768 06/30 Read block 91 len 20480 07/30 Read block 134 len 8192 08/30 Write block 313 len 24576 data: 11 7 7 9 11 11 09/30 Read block 166 len 4096 10/30 Read block 325 len 20480 11/30 Read block 537 len 40960 12/30 Write block 364 len 16384 data: 13 10 12 11 13/30 Read block 16 len 28672 14/30 Write block 587 len 4096 data: 4 15/30 Read block 171 len 36864 16/30 Write block 144 len 8192 data: 6 9 17/30 Write block 228 len 24576 data: 13 14 11 12 11 10 18/30 Write block 253 len 24576 data: 9 7 5 8 8 7 19/30 Write block 312 len 12288 data: 9 12 8 20/30 Read block 182 len 36864 21/30 Read block 193 len 20480 22/30 Write block 209 len 40960 data: 10 11 14 13 13 10 10 12 10 9 23/30 Write block 103 len 40960 data: 9 9 10 10 7 7 7 8 12 13 24/30 Write block 502 len 24576 data: 11 13 12 11 12 12 25/30 Flush 26/30 Write block 559 len 36864 data: 11 11 11 11 10 11 6 5 6 27/30 Write block 337 len 40960 data: 10 10 12 11 9 8 8 8 10 8 28/30 Write block 497 len 16384 data: 13 11 13 12 29/30 Read block 428 len 12288 30/30 Write block 128 len 24576 data: 9 9 11 8 8 8 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:07.558545556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 120 120 120 193 193 193 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 122 122 122 201 201 201 F F F 6 120-139 118 118 118 190 190 190 T T T 7 140-159 122 122 122 201 201 201 F F F 8 160-179 120 120 120 198 198 198 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 122 122 122 201 201 201 F F F 11 220-239 122 122 122 201 201 201 F F F 12 240-259 122 122 122 201 201 201 F F F 13 260-279 121 121 121 200 200 200 F F F 14 280-299 121 121 121 200 200 200 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 121 121 121 200 200 200 T T T 17 340-359 119 119 119 191 191 191 T T T 18 360-379 122 122 122 201 201 201 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 119 119 119 191 191 191 F F F 24 480-499 119 119 119 191 191 191 T T T 25 500-519 122 122 122 201 201 201 T T T 26 520-539 119 119 119 191 191 191 F F F 27 540-559 120 120 120 198 198 198 T T T 28 560-579 101 101 101 164 164 164 T T T 29 580-599 122 122 122 201 201 201 F F F Max gen: 122, Max flush: 201 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 120 120 120 193 193 193 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 122 122 122 201 201 201 F F F 6 120-139 123 123 123 202 202 202 F F F 7 140-159 122 122 122 201 201 201 F F F 8 160-179 120 120 120 198 198 198 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 122 122 122 201 201 201 F F F 11 220-239 122 122 122 201 201 201 F F F 12 240-259 122 122 122 201 201 201 F F F 13 260-279 121 121 121 200 200 200 F F F 14 280-299 121 121 121 200 200 200 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 123 123 123 202 202 202 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 122 122 122 201 201 201 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 F F F 21 420-439 115 115 115 186 186 186 F F F 22 440-459 115 115 115 186 186 186 F F F 23 460-479 119 119 119 191 191 191 F F F 24 480-499 123 123 123 202 202 202 F F F 25 500-519 123 123 123 202 202 202 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 123 123 123 202 202 202 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 122 122 122 201 201 201 F F F Max gen: 123, Max flush: 202 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.808038231Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:11.850442052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.850899275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.882420398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.883171135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88320839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.883223018Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.883237686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} {"msg":"Crucible a91caad4-5eb6-4339-9520-91a233fbb580 has session id: 8eb8be6e-2464-4551-9336-9262c03333d9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.883651105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Upstairs opts: Upstairs UUID: a91caad4-5eb6-4339-9520-91a233fbb580, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88369363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Crucible stats registered with UUID: a91caad4-5eb6-4339-9520-91a233fbb580","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.883719017Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:11.883738074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884187308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884217065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} {"msg":"a91caad4-5eb6-4339-9520-91a233fbb580 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884279587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884298145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884321732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88433679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884351618Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884365726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884380084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884393853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884408521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884422939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62888)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884732211Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60166)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884774306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37336)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.884799703Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88530429Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.885334646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.885355183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.885370141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88538369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.885397208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.885911364Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886058856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886216806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886517809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886547525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886563693Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886627245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88666796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886713295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886776397Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886801144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[0]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 201, 202, 201, 198, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886820941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[0]R generation[0..12]: [119, 120, 116, 119, 119, 122, 123, 122, 120, 121, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88683561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886850188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[1]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 201, 202, 201, 198, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886864386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[1]R generation[0..12]: [119, 120, 116, 119, 119, 122, 123, 122, 120, 121, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886878444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886892512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[2]R flush_numbers[0..12]: [191, 193, 187, 191, 191, 201, 202, 201, 198, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886906381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[2]R generation[0..12]: [119, 120, 116, 119, 119, 122, 123, 122, 120, 121, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886920219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886934147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886947256Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886960584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.886973822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-11T23:46:11.886987111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"":"downstairs","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887006578Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887021666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"0","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887035105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"1","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887048403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"client":"2","":"downstairs","session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887061671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.88707399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"a91caad4-5eb6-4339-9520-91a233fbb580 is now active with session: 8eb8be6e-2464-4551-9336-9262c03333d9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887086878Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887099117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678,"session_id":"8eb8be6e-2464-4551-9336-9262c03333d9"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:11.887111485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 24576 data: 14 14 14 13 11 14 02/30 Write block 460 len 36864 data: 10 9 9 7 8 8 9 8 8 03/30 Flush 04/30 Write block 23 len 8192 data: 18 15 05/30 Read block 185 len 20480 06/30 Read block 253 len 20480 07/30 Write block 463 len 20480 data: 8 9 9 10 9 08/30 Read block 416 len 4096 09/30 Flush 10/30 Write block 424 len 20480 data: 15 15 14 12 15 11/30 Read block 150 len 16384 12/30 Read block 166 len 4096 13/30 Write block 439 len 24576 data: 15 13 10 12 12 12 14/30 Read block 260 len 36864 15/30 Read block 487 len 4096 16/30 Read block 488 len 12288 17/30 Read block 352 len 12288 18/30 Flush 19/30 Write block 584 len 20480 data: 4 4 4 5 5 20/30 Flush 21/30 Flush 22/30 Write block 238 len 20480 data: 8 8 8 8 8 23/30 Read block 434 len 8192 24/30 Write block 275 len 4096 data: 9 25/30 Write block 141 len 32768 data: 8 6 6 7 10 9 8 10 26/30 Read block 407 len 36864 27/30 Write block 415 len 28672 data: 14 12 14 11 11 10 13 28/30 Read block 464 len 12288 29/30 Read block 129 len 24576 30/30 Read block 61 len 32768 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:11.90678854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 122 122 122 201 201 201 F F F 6 120-139 123 123 123 202 202 202 F F F 7 140-159 122 122 122 201 201 201 T T T 8 160-179 120 120 120 198 198 198 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 122 122 122 201 201 201 F F F 11 220-239 122 122 122 201 201 201 T T T 12 240-259 122 122 122 201 201 201 T T T 13 260-279 121 121 121 200 200 200 T T T 14 280-299 121 121 121 200 200 200 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 123 123 123 202 202 202 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 122 122 122 201 201 201 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 118 118 118 190 190 190 T T T 21 420-439 124 124 124 205 205 205 T T T 22 440-459 124 124 124 205 205 205 F F F 23 460-479 124 124 124 204 204 204 F F F 24 480-499 123 123 123 202 202 202 F F F 25 500-519 123 123 123 202 202 202 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 123 123 123 202 202 202 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 124 124 124 206 206 206 F F F Max gen: 124, Max flush: 206 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 122 122 122 201 201 201 F F F 6 120-139 123 123 123 202 202 202 F F F 7 140-159 125 125 125 207 207 207 F F F 8 160-179 120 120 120 198 198 198 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 122 122 122 201 201 201 F F F 11 220-239 125 125 125 207 207 207 F F F 12 240-259 125 125 125 207 207 207 F F F 13 260-279 125 125 125 207 207 207 F F F 14 280-299 121 121 121 200 200 200 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 123 123 123 202 202 202 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 122 122 122 201 201 201 F F F 19 380-399 119 119 119 191 191 191 F F F 20 400-419 125 125 125 207 207 207 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 124 124 124 205 205 205 F F F 23 460-479 124 124 124 204 204 204 F F F 24 480-499 123 123 123 202 202 202 F F F 25 500-519 123 123 123 202 202 202 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 123 123 123 202 202 202 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 124 124 124 206 206 206 F F F Max gen: 125, Max flush: 207 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.175214123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:16.217359105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.217753736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.248473539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.249124728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.249162253Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.249176822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.24919194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} {"msg":"Crucible 3ab80485-4a5b-49f6-9ade-5eaf19d2bb2d has session id: 4ea40cbc-03b7-43c2-bef7-57d918ede462","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.249706936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Upstairs opts: Upstairs UUID: 3ab80485-4a5b-49f6-9ade-5eaf19d2bb2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.249737232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Crucible stats registered with UUID: 3ab80485-4a5b-49f6-9ade-5eaf19d2bb2d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.24975324Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:16.249766848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250214933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250240959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} {"msg":"3ab80485-4a5b-49f6-9ade-5eaf19d2bb2d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250286074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250303111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250326439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250341497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250360604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250419977Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250482409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250524884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25055485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250584017Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_connection connected from Ok(127.0.0.1:35444)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250727869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_connection connected from Ok(127.0.0.1:44979)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.250771343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_connection connected from Ok(127.0.0.1:64145)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25079785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251347482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251376668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251395316Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251425162Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251451159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.251475886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252131704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252161751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252177589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252707973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25273228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252746648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252759346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252772735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252785963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252798061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25281267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[0]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 201, 202, 207, 198, 200, 201, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252826918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[0]R generation[0..12]: [119, 124, 116, 119, 119, 122, 123, 125, 120, 121, 122, 125]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252845176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252859784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[1]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 201, 202, 207, 198, 200, 201, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252873652Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[1]R generation[0..12]: [119, 124, 116, 119, 119, 122, 123, 125, 120, 121, 122, 125]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.2528875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252901749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[2]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 201, 202, 207, 198, 200, 201, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252915767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[2]R generation[0..12]: [119, 124, 116, 119, 119, 122, 123, 125, 120, 121, 122, 125]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252929565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252943414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252956582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25296996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252983069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.252996047Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"":"downstairs","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253009325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:46:16.253021514Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"0","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253040381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"1","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.25305472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"client":"2","":"downstairs","session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253068388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253081646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"3ab80485-4a5b-49f6-9ade-5eaf19d2bb2d is now active with session: 4ea40cbc-03b7-43c2-bef7-57d918ede462","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253094795Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253106793Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686,"session_id":"4ea40cbc-03b7-43c2-bef7-57d918ede462"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:16.253119192Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 12288 data: 8 8 8 02/30 Read block 456 len 40960 03/30 Write block 441 len 40960 data: 11 13 13 13 10 10 12 7 7 9 04/30 Read block 86 len 24576 05/30 Write block 114 len 12288 data: 15 15 14 06/30 Read block 162 len 4096 07/30 Read block 220 len 28672 08/30 Write block 152 len 36864 data: 16 17 17 17 17 16 11 6 8 09/30 Read block 418 len 24576 10/30 Write block 275 len 28672 data: 10 9 8 9 8 7 8 11/30 Read block 166 len 16384 12/30 Flush 13/30 Write block 138 len 12288 data: 10 10 7 14/30 Write block 167 len 8192 data: 13 10 15/30 Write block 470 len 16384 data: 6 6 10 9 16/30 Read block 564 len 28672 17/30 Write block 594 len 12288 data: 11 9 6 18/30 Read block 425 len 16384 19/30 Write block 557 len 12288 data: 9 10 12 20/30 Read block 150 len 20480 21/30 Read block 348 len 32768 22/30 Read block 449 len 28672 23/30 Read block 19 len 32768 24/30 Write block 169 len 4096 data: 9 25/30 Write block 210 len 32768 data: 12 15 14 14 11 11 13 11 26/30 Write block 220 len 28672 data: 8 8 9 10 10 11 13 27/30 Write block 451 len 40960 data: 8 11 12 11 9 10 10 7 9 11 28/30 Write block 598 len 8192 data: 4 3 29/30 Read block 526 len 32768 30/30 Read block 383 len 8192 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:16.273383873Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 123 123 123 202 202 202 T T T 7 140-159 126 126 126 208 208 208 T T T 8 160-179 126 126 126 208 208 208 T T T 9 180-199 121 121 121 200 200 200 F F F 10 200-219 122 122 122 201 201 201 T T T 11 220-239 125 125 125 207 207 207 T T T 12 240-259 125 125 125 207 207 207 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 126 126 126 208 208 208 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 123 123 123 202 202 202 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 126 126 126 208 208 208 F F F 20 400-419 125 125 125 207 207 207 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 126 126 126 208 208 208 T T T 23 460-479 124 124 124 204 204 204 T T T 24 480-499 123 123 123 202 202 202 F F F 25 500-519 123 123 123 202 202 202 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 123 123 123 202 202 202 T T T 28 560-579 123 123 123 202 202 202 F F F 29 580-599 124 124 124 206 206 206 T T T Max gen: 126, Max flush: 208 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 116 116 116 187 187 187 F F F 3 060-079 119 119 119 191 191 191 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 127 127 127 209 209 209 F F F 7 140-159 127 127 127 209 209 209 F F F 8 160-179 127 127 127 209 209 209 F F F 9 180-199 121 121 121 200 200 200 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 125 125 125 207 207 207 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 126 126 126 208 208 208 F F F 15 300-319 122 122 122 201 201 201 F F F 16 320-339 123 123 123 202 202 202 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 126 126 126 208 208 208 F F F 19 380-399 126 126 126 208 208 208 F F F 20 400-419 125 125 125 207 207 207 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 127 127 127 209 209 209 F F F 23 460-479 127 127 127 209 209 209 F F F 24 480-499 123 123 123 202 202 202 F F F 25 500-519 123 123 123 202 202 202 F F F 26 520-539 119 119 119 191 191 191 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 127 127 127 209 209 209 F F F Max gen: 127, Max flush: 209 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.565876865Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:20.607888605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.608308082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.638964013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.639525123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.639561529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.639575907Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.639591015Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} {"msg":"Crucible d6a464d2-0606-44e8-b31d-f7ccef7b0249 has session id: d29eb268-3120-4a74-9a1c-42afb32d2c21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.64003096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Upstairs opts: Upstairs UUID: d6a464d2-0606-44e8-b31d-f7ccef7b0249, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640075245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Crucible stats registered with UUID: d6a464d2-0606-44e8-b31d-f7ccef7b0249","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640102361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:20.640129118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640520929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640545366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} {"msg":"d6a464d2-0606-44e8-b31d-f7ccef7b0249 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640631266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640650573Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.64067322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640734203Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640800155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640825272Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.640850738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641048934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641087739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641145812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:34918)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641200265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:41700)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641234521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:48024)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641264487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641542352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.64164405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641684225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641793191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641821398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.641836886Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642236686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642265813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642281971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642795137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642819684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642833862Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642890525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642920691Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642936199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642949357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642967915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[0]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 208, 209, 209, 209, 200, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.642982544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[0]R generation[0..12]: [119, 124, 116, 119, 119, 126, 127, 127, 127, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643000151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643024848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[1]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 208, 209, 209, 209, 200, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643040646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[1]R generation[0..12]: [119, 124, 116, 119, 119, 126, 127, 127, 127, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643054755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643068943Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[2]R flush_numbers[0..12]: [191, 204, 187, 191, 191, 208, 209, 209, 209, 200, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643082891Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[2]R generation[0..12]: [119, 124, 116, 119, 119, 126, 127, 127, 127, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643096719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643110737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643123796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643137124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643150153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"All extents match","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.64316936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"":"downstairs","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643186028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643198626Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"0","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643212145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"1","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643225513Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"client":"2","":"downstairs","session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643238921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.64325137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"d6a464d2-0606-44e8-b31d-f7ccef7b0249 is now active with session: d29eb268-3120-4a74-9a1c-42afb32d2c21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643264369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643276567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694,"session_id":"d29eb268-3120-4a74-9a1c-42afb32d2c21"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:20.643288905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 383 len 4096 data: 7 02/30 Write block 367 len 24576 data: 12 10 12 12 12 11 03/30 Read block 491 len 16384 04/30 Read block 109 len 20480 05/30 Read block 265 len 36864 06/30 Write block 55 len 40960 data: 9 10 10 10 14 11 14 14 10 13 07/30 Read block 485 len 36864 08/30 Write block 313 len 32768 data: 13 9 8 10 12 12 12 13 09/30 Read block 118 len 20480 10/30 Write block 180 len 16384 data: 14 13 11 12 11/30 Read block 535 len 40960 12/30 Read block 165 len 20480 13/30 Write block 300 len 8192 data: 14 9 14/30 Flush 15/30 Write block 459 len 28672 data: 10 12 10 10 9 10 10 16/30 Write block 139 len 32768 data: 11 8 9 7 7 8 11 10 17/30 Read block 493 len 28672 18/30 Read block 251 len 24576 19/30 Read block 73 len 20480 20/30 Read block 203 len 20480 21/30 Read block 239 len 4096 22/30 Flush 23/30 Write block 399 len 4096 data: 15 24/30 Write block 504 len 20480 data: 13 12 13 13 13 25/30 Write block 489 len 8192 data: 10 10 26/30 Write block 527 len 12288 data: 7 5 5 27/30 Read block 343 len 20480 28/30 Read block 512 len 36864 29/30 Write block 22 len 32768 data: 17 19 16 16 14 15 13 14 30/30 Write block 247 len 16384 data: 7 8 9 11 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:20.661873656Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 124 124 124 204 204 204 T T T 2 040-059 128 128 128 210 210 210 F F F 3 060-079 128 128 128 210 210 210 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 128 128 128 211 211 211 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 127 127 127 209 209 209 F F F 9 180-199 128 128 128 210 210 210 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 125 125 125 207 207 207 T T T 13 260-279 126 126 126 208 208 208 F F F 14 280-299 126 126 126 208 208 208 F F F 15 300-319 128 128 128 210 210 210 F F F 16 320-339 128 128 128 210 210 210 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 128 128 128 210 210 210 T T T 20 400-419 125 125 125 207 207 207 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 128 128 128 211 211 211 F F F 23 460-479 128 128 128 211 211 211 F F F 24 480-499 123 123 123 202 202 202 T T T 25 500-519 123 123 123 202 202 202 T T T 26 520-539 119 119 119 191 191 191 T T T 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 127 127 127 209 209 209 F F F Max gen: 128, Max flush: 211 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 F F F 1 020-039 129 129 129 212 212 212 F F F 2 040-059 128 128 128 210 210 210 F F F 3 060-079 128 128 128 210 210 210 F F F 4 080-099 119 119 119 191 191 191 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 128 128 128 211 211 211 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 127 127 127 209 209 209 F F F 9 180-199 128 128 128 210 210 210 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 126 126 126 208 208 208 F F F 15 300-319 128 128 128 210 210 210 F F F 16 320-339 128 128 128 210 210 210 F F F 17 340-359 123 123 123 202 202 202 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 F F F 20 400-419 125 125 125 207 207 207 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 128 128 128 211 211 211 F F F 23 460-479 128 128 128 211 211 211 F F F 24 480-499 129 129 129 212 212 212 F F F 25 500-519 129 129 129 212 212 212 F F F 26 520-539 129 129 129 212 212 212 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 127 127 127 209 209 209 F F F Max gen: 129, Max flush: 212 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:24.938608106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:24.980285716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:24.980689536Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.011604235Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012291029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012326885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012341403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012356641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} {"msg":"Crucible df732805-cab6-4531-b75d-289894840d2d has session id: 9089454c-acbe-4780-ac7e-ef1b49c6c121","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012778879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Upstairs opts: Upstairs UUID: df732805-cab6-4531-b75d-289894840d2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012839231Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Crucible stats registered with UUID: df732805-cab6-4531-b75d-289894840d2d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.012865428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:25.012883785Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01332978Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013356577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} {"msg":"df732805-cab6-4531-b75d-289894840d2d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013398322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01341507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013438387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013453455Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013467963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013482061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013496329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013510278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013524936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013539594Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_connection connected from Ok(127.0.0.1:38881)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013791053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_connection connected from Ok(127.0.0.1:47147)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013836517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_connection connected from Ok(127.0.0.1:58481)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.013866603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014401367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014440392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014521502Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014555338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014591333Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.014630599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01510534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015158433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015192989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015681698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015715314Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015731282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01582726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015860516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015887952Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01590279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015920588Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[0]R flush_numbers[0..12]: [191, 212, 210, 210, 191, 208, 211, 211, 209, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015935196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[0]R generation[0..12]: [119, 129, 128, 128, 119, 126, 128, 128, 127, 128, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015949495Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015963743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[1]R flush_numbers[0..12]: [191, 212, 210, 210, 191, 208, 211, 211, 209, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015977961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[1]R generation[0..12]: [119, 129, 128, 128, 119, 126, 128, 128, 127, 128, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.015991989Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016006157Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[2]R flush_numbers[0..12]: [191, 212, 210, 210, 191, 208, 211, 211, 209, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016020366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[2]R generation[0..12]: [119, 129, 128, 128, 119, 126, 128, 128, 127, 128, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016034324Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016048702Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016061971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016075369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016088587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016101826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"":"downstairs","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016115634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-12-11T23:46:25.016136461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"0","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01615043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"1","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016164038Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"client":"2","":"downstairs","session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016177516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016189965Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"df732805-cab6-4531-b75d-289894840d2d is now active with session: 9089454c-acbe-4780-ac7e-ef1b49c6c121","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016203163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.016215462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702,"session_id":"9089454c-acbe-4780-ac7e-ef1b49c6c121"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:25.01622804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 4096 data: 15 02/30 Read block 294 len 16384 03/30 Read block 351 len 4096 04/30 Write block 84 len 24576 data: 11 12 14 11 13 14 05/30 Flush 06/30 Write block 24 len 20480 data: 17 17 15 16 14 07/30 Write block 178 len 16384 data: 15 14 15 14 08/30 Write block 22 len 12288 data: 18 20 18 09/30 Read block 505 len 24576 10/30 Read block 326 len 16384 11/30 Read block 516 len 8192 12/30 Write block 338 len 40960 data: 11 13 12 10 9 9 9 11 9 8 13/30 Read block 578 len 28672 14/30 Read block 240 len 12288 15/30 Write block 290 len 12288 data: 9 10 9 16/30 Flush 17/30 Write block 185 len 40960 data: 10 10 8 10 8 7 8 8 11 13 18/30 Read block 256 len 24576 19/30 Read block 551 len 24576 20/30 Write block 68 len 40960 data: 12 8 10 12 11 11 10 10 8 8 21/30 Write block 472 len 16384 data: 11 10 9 12 22/30 Write block 494 len 24576 data: 10 14 15 14 12 14 23/30 Read block 588 len 32768 24/30 Read block 484 len 40960 25/30 Read block 564 len 12288 26/30 Read block 574 len 16384 27/30 Write block 480 len 32768 data: 15 16 14 14 10 10 9 10 28/30 Read block 440 len 28672 29/30 Read block 508 len 8192 30/30 Write block 3 len 20480 data: 11 10 11 14 16 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:25.036407682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 191 191 191 T T T 1 020-039 130 130 130 214 214 214 F F F 2 040-059 128 128 128 210 210 210 F F F 3 060-079 128 128 128 210 210 210 T T T 4 080-099 130 130 130 213 213 213 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 128 128 128 211 211 211 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 130 130 130 214 214 214 T T T 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 128 128 128 210 210 210 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 130 130 130 214 214 214 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 F F F 20 400-419 130 130 130 213 213 213 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 128 128 128 211 211 211 F F F 23 460-479 128 128 128 211 211 211 T T T 24 480-499 129 129 129 212 212 212 T T T 25 500-519 129 129 129 212 212 212 F F F 26 520-539 129 129 129 212 212 212 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 127 127 127 209 209 209 F F F Max gen: 130, Max flush: 214 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 215 215 215 F F F 1 020-039 130 130 130 214 214 214 F F F 2 040-059 128 128 128 210 210 210 F F F 3 060-079 131 131 131 215 215 215 F F F 4 080-099 130 130 130 213 213 213 F F F 5 100-119 126 126 126 208 208 208 F F F 6 120-139 128 128 128 211 211 211 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 131 131 131 215 215 215 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 128 128 128 210 210 210 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 130 130 130 214 214 214 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 F F F 20 400-419 130 130 130 213 213 213 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 128 128 128 211 211 211 F F F 23 460-479 131 131 131 215 215 215 F F F 24 480-499 131 131 131 215 215 215 F F F 25 500-519 129 129 129 212 212 212 F F F 26 520-539 129 129 129 212 212 212 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 127 127 127 209 209 209 F F F Max gen: 131, Max flush: 215 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.311465701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.354971814Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.355704323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.355741509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.355756237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.355771365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} {"msg":"Crucible 9fc15250-86cb-495f-9afb-82d563be4bee has session id: 0be5f5c8-91a3-4938-91ed-1124491fc471","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356286671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Upstairs opts: Upstairs UUID: 9fc15250-86cb-495f-9afb-82d563be4bee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356329746Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Crucible stats registered with UUID: 9fc15250-86cb-495f-9afb-82d563be4bee","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356350603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:29.356364361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356809776Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356840792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} {"msg":"9fc15250-86cb-495f-9afb-82d563be4bee active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356870119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356883847Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356907884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356929671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356957448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.356984524Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357008241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357030779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357054876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357082302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_connection connected from Ok(127.0.0.1:60956)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.35726343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_connection connected from Ok(127.0.0.1:56194)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357302755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_connection connected from Ok(127.0.0.1:36614)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357326112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357864035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.35790483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357941376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.357972892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.358235519Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.358264116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.358563548Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.358755744Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.358960279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359158184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359189821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359205088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359217917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359231515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359244783Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359452518Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359476865Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[0]R flush_numbers[0..12]: [215, 214, 210, 215, 213, 208, 211, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359495392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[0]R generation[0..12]: [131, 130, 128, 131, 130, 126, 128, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359510061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359524379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[1]R flush_numbers[0..12]: [215, 214, 210, 215, 213, 208, 211, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359538287Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[1]R generation[0..12]: [131, 130, 128, 131, 130, 126, 128, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359552065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359566114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[2]R flush_numbers[0..12]: [215, 214, 210, 215, 213, 208, 211, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359579872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[2]R generation[0..12]: [131, 130, 128, 131, 130, 126, 128, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.35959349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359607709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359620917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359634435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359647773Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:46:29.359661102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"":"downstairs","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.35967725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359691598Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"0","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359705166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"1","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359718355Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"client":"2","":"downstairs","session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359731543Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359743941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"9fc15250-86cb-495f-9afb-82d563be4bee is now active with session: 0be5f5c8-91a3-4938-91ed-1124491fc471","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.35975704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359769178Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710,"session_id":"0be5f5c8-91a3-4938-91ed-1124491fc471"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:29.359781637Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 584 len 40960 data: 5 5 5 6 6 8 10 11 10 10 03/30 Read block 567 len 36864 04/30 Flush 05/30 Flush 06/30 Write block 79 len 24576 data: 8 8 7 9 10 12 07/30 Write block 334 len 32768 data: 6 7 8 11 12 14 13 11 08/30 Read block 285 len 20480 09/30 Write block 482 len 12288 data: 15 15 11 10/30 Read block 245 len 4096 11/30 Flush 12/30 Flush 13/30 Write block 446 len 20480 data: 11 13 8 8 10 14/30 Write block 470 len 40960 data: 7 7 12 11 10 13 13 14 15 15 15/30 Write block 300 len 32768 data: 15 10 9 9 8 7 7 6 16/30 Write block 39 len 32768 data: 11 10 8 11 14 12 12 11 17/30 Read block 513 len 4096 18/30 Write block 328 len 20480 data: 13 10 8 7 6 19/30 Read block 404 len 40960 20/30 Read block 592 len 24576 21/30 Write block 37 len 20480 data: 10 10 12 11 9 22/30 Flush 23/30 Write block 63 len 12288 data: 11 14 12 24/30 Read block 156 len 12288 25/30 Read block 249 len 8192 26/30 Read block 261 len 4096 27/30 Write block 100 len 40960 data: 8 10 10 10 10 11 11 8 8 8 28/30 Write block 335 len 40960 data: 8 9 12 13 15 14 12 10 10 10 29/30 Read block 172 len 4096 30/30 Write block 124 len 20480 data: 8 7 8 10 10 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:29.379509155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 215 215 215 F F F 1 020-039 132 132 132 221 221 221 F F F 2 040-059 132 132 132 221 221 221 F F F 3 060-079 132 132 132 219 219 219 T T T 4 080-099 132 132 132 219 219 219 F F F 5 100-119 126 126 126 208 208 208 T T T 6 120-139 128 128 128 211 211 211 T T T 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 131 131 131 215 215 215 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 132 132 132 221 221 221 T T T 17 340-359 132 132 132 219 219 219 T T T 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 F F F 20 400-419 130 130 130 213 213 213 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 132 132 132 221 221 221 F F F 23 460-479 132 132 132 221 221 221 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 129 129 129 212 212 212 F F F 26 520-539 129 129 129 212 212 212 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 132 132 132 217 217 217 F F F Max gen: 132, Max flush: 221 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 215 215 215 F F F 1 020-039 132 132 132 221 221 221 F F F 2 040-059 132 132 132 221 221 221 F F F 3 060-079 133 133 133 222 222 222 F F F 4 080-099 132 132 132 219 219 219 F F F 5 100-119 133 133 133 222 222 222 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 131 131 131 215 215 215 F F F 10 200-219 127 127 127 209 209 209 F F F 11 220-239 127 127 127 209 209 209 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 F F F 20 400-419 130 130 130 213 213 213 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 132 132 132 221 221 221 F F F 23 460-479 132 132 132 221 221 221 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 129 129 129 212 212 212 F F F 26 520-539 129 129 129 212 212 212 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 132 132 132 217 217 217 F F F Max gen: 133, Max flush: 222 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.620370953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:33.663247285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.663633307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.695324489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.695985557Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.69604113Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.696056028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.696071016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} {"msg":"Crucible b9768a91-a833-4df7-8e55-773f91311850 has session id: 9cca06ff-0adc-441f-beeb-18aeb5dcd139","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.696534019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Upstairs opts: Upstairs UUID: b9768a91-a833-4df7-8e55-773f91311850, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.696566624Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Crucible stats registered with UUID: b9768a91-a833-4df7-8e55-773f91311850","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.696582522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:33.696595731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697111157Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697135904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} {"msg":"b9768a91-a833-4df7-8e55-773f91311850 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697198446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697218394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.69724872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697277266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697306363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697335479Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697363016Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697392402Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697420758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697446635Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_connection connected from Ok(127.0.0.1:60319)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697473482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_connection connected from Ok(127.0.0.1:64405)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697501538Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_connection connected from Ok(127.0.0.1:63798)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.697528995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698268293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698307208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698343834Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698410145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698444461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.698472458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699025209Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699055685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699074893Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699594678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699633173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699652391Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699730571Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699750759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699764547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699776896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699790344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[0]R flush_numbers[0..12]: [215, 221, 221, 222, 219, 222, 222, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699817171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[0]R generation[0..12]: [131, 132, 132, 133, 132, 133, 133, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699835528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699850126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[1]R flush_numbers[0..12]: [215, 221, 221, 222, 219, 222, 222, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699864125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[1]R generation[0..12]: [131, 132, 132, 133, 132, 133, 133, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699877933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699892071Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[2]R flush_numbers[0..12]: [215, 221, 221, 222, 219, 222, 222, 211, 214, 215, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.6999063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[2]R generation[0..12]: [131, 132, 132, 133, 132, 133, 133, 128, 130, 131, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699920508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699934676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699947824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699961053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699974121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.699987199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"":"downstairs","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700000478Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700012636Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"0","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700028574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"1","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700041952Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"client":"2","":"downstairs","session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.70006146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700074088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"b9768a91-a833-4df7-8e55-773f91311850 is now active with session: 9cca06ff-0adc-441f-beeb-18aeb5dcd139","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700087037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700099405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718,"session_id":"9cca06ff-0adc-441f-beeb-18aeb5dcd139"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:33.700111854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 465 len 8192 data: 11 11 02/30 Read block 247 len 16384 03/30 Read block 90 len 4096 04/30 Read block 437 len 40960 05/30 Read block 17 len 36864 06/30 Write block 460 len 4096 data: 13 07/30 Flush 08/30 Write block 87 len 8192 data: 12 14 09/30 Flush 10/30 Write block 522 len 24576 data: 8 9 7 6 7 8 11/30 Read block 350 len 16384 12/30 Write block 532 len 20480 data: 10 11 10 12 12 13/30 Write block 91 len 28672 data: 7 7 5 5 4 6 7 14/30 Read block 432 len 40960 15/30 Write block 519 len 28672 data: 9 8 10 9 10 8 7 16/30 Read block 63 len 8192 17/30 Write block 394 len 28672 data: 9 14 14 14 14 16 13 18/30 Write block 217 len 24576 data: 12 10 5 9 9 10 19/30 Write block 16 len 28672 data: 14 14 17 18 16 15 19 20/30 Write block 450 len 4096 data: 11 21/30 Write block 462 len 36864 data: 11 10 11 12 12 10 9 8 8 22/30 Write block 391 len 24576 data: 10 8 9 10 15 15 23/30 Read block 537 len 20480 24/30 Write block 477 len 4096 data: 15 25/30 Read block 81 len 36864 26/30 Read block 221 len 16384 27/30 Write block 584 len 4096 data: 6 28/30 Read block 380 len 24576 29/30 Write block 40 len 8192 data: 12 10 30/30 Read block 90 len 20480 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 6 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 21 20 21 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:33.718865613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 215 215 215 T T T 1 020-039 132 132 132 221 221 221 T T T 2 040-059 132 132 132 221 221 221 T T T 3 060-079 133 133 133 222 222 222 F F F 4 080-099 134 134 134 224 224 224 T T T 5 100-119 133 133 133 222 222 222 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 131 131 131 215 215 215 F F F 10 200-219 127 127 127 209 209 209 T T T 11 220-239 127 127 127 209 209 209 T T T 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 129 129 129 212 212 212 T T T 20 400-419 130 130 130 213 213 213 T T T 21 420-439 125 125 125 207 207 207 F F F 22 440-459 132 132 132 221 221 221 T T T 23 460-479 134 134 134 223 223 223 T T T 24 480-499 132 132 132 219 219 219 F F F 25 500-519 129 129 129 212 212 212 T T T 26 520-539 129 129 129 212 212 212 T T T 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 132 132 132 217 217 217 T T T Max gen: 134, Max flush: 224 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 225 225 225 F F F 1 020-039 135 135 135 225 225 225 F F F 2 040-059 135 135 135 225 225 225 F F F 3 060-079 133 133 133 222 222 222 F F F 4 080-099 135 135 135 225 225 225 F F F 5 100-119 133 133 133 222 222 222 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 F F F 9 180-199 131 131 131 215 215 215 F F F 10 200-219 135 135 135 225 225 225 F F F 11 220-239 135 135 135 225 225 225 F F F 12 240-259 129 129 129 212 212 212 F F F 13 260-279 126 126 126 208 208 208 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 135 135 135 225 225 225 F F F 23 460-479 135 135 135 225 225 225 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 135 135 135 225 225 225 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 135 135 135 225 225 225 F F F Max gen: 135, Max flush: 225 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.036241503Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:38.077836084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.078232764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.109065083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.1096484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.109684456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.109698294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.109713132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} {"msg":"Crucible dffa2dd4-02d4-4f6f-a03a-5fd549b0b9ef has session id: d371fb67-c74d-463d-8840-765be4b73493","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110253355Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Upstairs opts: Upstairs UUID: dffa2dd4-02d4-4f6f-a03a-5fd549b0b9ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110291821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Crucible stats registered with UUID: dffa2dd4-02d4-4f6f-a03a-5fd549b0b9ef","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110308628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:38.110325796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110732136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110756173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} {"msg":"dffa2dd4-02d4-4f6f-a03a-5fd549b0b9ef active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110804127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110832173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.11085768Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110873308Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110888976Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110952658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110978495Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.110995233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111010411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111024639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_connection connected from Ok(127.0.0.1:54584)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111038828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_connection connected from Ok(127.0.0.1:59104)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111224705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_connection connected from Ok(127.0.0.1:37225)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111265909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.11158239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111623175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111757248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111792544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111871484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.111910609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112298301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112334547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112442783Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112838054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.1128743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112901116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112921963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112936582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.11295009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.112982106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113012162Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 225, 222, 225, 222, 222, 211, 214, 215, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.11302898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[0]R generation[0..12]: [135, 135, 135, 133, 135, 133, 133, 128, 130, 131, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113043199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113057607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 225, 222, 225, 222, 222, 211, 214, 215, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113071435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[1]R generation[0..12]: [135, 135, 135, 133, 135, 133, 133, 128, 130, 131, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113085263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113099381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 225, 222, 225, 222, 222, 211, 214, 215, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.11311306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[2]R generation[0..12]: [135, 135, 135, 133, 135, 133, 133, 128, 130, 131, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113126758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113140996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113153955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113167153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113180302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113196069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"":"downstairs","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113209808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113222226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"0","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113235755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"1","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113249173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"client":"2","":"downstairs","session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:46:38.113262591Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113280069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"dffa2dd4-02d4-4f6f-a03a-5fd549b0b9ef is now active with session: d371fb67-c74d-463d-8840-765be4b73493","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113293077Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113305266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726,"session_id":"d371fb67-c74d-463d-8840-765be4b73493"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:38.113317774Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 32768 data: 10 8 8 6 6 5 7 8 02/30 Read block 84 len 4096 03/30 Read block 376 len 4096 04/30 Write block 214 len 4096 data: 12 05/30 Read block 447 len 32768 06/30 Write block 112 len 16384 data: 14 14 16 16 07/30 Read block 283 len 28672 08/30 Read block 591 len 8192 09/30 Read block 14 len 32768 10/30 Read block 532 len 20480 11/30 Write block 236 len 4096 data: 10 12/30 Read block 410 len 20480 13/30 Write block 580 len 16384 data: 3 4 5 3 14/30 Write block 261 len 24576 data: 9 10 10 11 11 9 15/30 Read block 33 len 16384 16/30 Read block 10 len 8192 17/30 Write block 1 len 24576 data: 6 9 12 11 12 15 18/30 Write block 264 len 16384 data: 12 12 10 10 19/30 Read block 158 len 4096 20/30 Write block 186 len 16384 data: 11 9 11 9 21/30 Write block 38 len 20480 data: 11 13 13 11 12 22/30 Read block 253 len 36864 23/30 Write block 229 len 36864 data: 15 12 13 12 11 11 11 11 7 24/30 Write block 174 len 24576 data: 11 8 14 13 16 15 25/30 Write block 58 len 20480 data: 11 15 12 15 15 26/30 Read block 9 len 20480 27/30 Write block 279 len 4096 data: 9 28/30 Read block 249 len 40960 29/30 Write block 244 len 28672 data: 5 5 5 8 9 10 12 30/30 Read block 242 len 16384 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 30 28 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:38.131793698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 225 225 225 T T T 1 020-039 135 135 135 225 225 225 T T T 2 040-059 135 135 135 225 225 225 T T T 3 060-079 133 133 133 222 222 222 T T T 4 080-099 135 135 135 225 225 225 T T T 5 100-119 133 133 133 222 222 222 T T T 6 120-139 133 133 133 222 222 222 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 130 130 130 214 214 214 T T T 9 180-199 131 131 131 215 215 215 T T T 10 200-219 135 135 135 225 225 225 T T T 11 220-239 135 135 135 225 225 225 T T T 12 240-259 129 129 129 212 212 212 T T T 13 260-279 126 126 126 208 208 208 T T T 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 135 135 135 225 225 225 F F F 23 460-479 135 135 135 225 225 225 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 135 135 135 225 225 225 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 135 135 135 225 225 225 T T T Max gen: 135, Max flush: 225 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 226 226 226 F F F 1 020-039 136 136 136 226 226 226 F F F 2 040-059 136 136 136 226 226 226 F F F 3 060-079 136 136 136 226 226 226 F F F 4 080-099 136 136 136 226 226 226 F F F 5 100-119 136 136 136 226 226 226 F F F 6 120-139 133 133 133 222 222 222 F F F 7 140-159 128 128 128 211 211 211 F F F 8 160-179 136 136 136 226 226 226 F F F 9 180-199 136 136 136 226 226 226 F F F 10 200-219 136 136 136 226 226 226 F F F 11 220-239 136 136 136 226 226 226 F F F 12 240-259 136 136 136 226 226 226 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 130 130 130 214 214 214 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 135 135 135 225 225 225 F F F 23 460-479 135 135 135 225 225 225 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 135 135 135 225 225 225 F F F 27 540-559 127 127 127 209 209 209 F F F 28 560-579 123 123 123 202 202 202 F F F 29 580-599 136 136 136 226 226 226 F F F Max gen: 136, Max flush: 226 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.446598848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.492000676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.492661484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.492699509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.492714057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.492729255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} {"msg":"Crucible 0bb95143-e627-4c23-8994-ef491c5a07a5 has session id: 5260f716-5a8d-4a26-8a15-33b277cd1e4b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493140004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Upstairs opts: Upstairs UUID: 0bb95143-e627-4c23-8994-ef491c5a07a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493192268Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Crucible stats registered with UUID: 0bb95143-e627-4c23-8994-ef491c5a07a5","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493219714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:42.493237512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493715463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493751488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} {"msg":"0bb95143-e627-4c23-8994-ef491c5a07a5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493809901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493836508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493871833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.49390134Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493930456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493959052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.493987579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494018295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494047732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494077958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54606)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494106224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52510)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.49413756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53899)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494167397Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494796468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494831224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.494848022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.49486208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.495492872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.495523228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.495601778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.495638014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496064411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496106126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496133102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.49614722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496161909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496179476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496195345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496691303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496720249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[0]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 222, 211, 226, 226, 226, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496735837Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[0]R generation[0..12]: [136, 136, 136, 136, 136, 136, 133, 128, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496750436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496764934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[1]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 222, 211, 226, 226, 226, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496779122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[1]R generation[0..12]: [136, 136, 136, 136, 136, 136, 133, 128, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496793Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496807569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[2]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 222, 211, 226, 226, 226, 226]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496821607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[2]R generation[0..12]: [136, 136, 136, 136, 136, 136, 133, 128, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496835345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496849233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496862392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.49687924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496892838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496905846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"":"downstairs","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:46:42.496919235Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496935522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"0","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496948961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"1","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496962189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"client":"2","":"downstairs","session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496975648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.496988076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"0bb95143-e627-4c23-8994-ef491c5a07a5 is now active with session: 5260f716-5a8d-4a26-8a15-33b277cd1e4b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.497008573Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.497029001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734,"session_id":"5260f716-5a8d-4a26-8a15-33b277cd1e4b"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:42.497042469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 40960 data: 20 21 19 18 16 17 15 15 13 12 02/30 Write block 95 len 4096 data: 6 03/30 Write block 108 len 24576 data: 9 9 9 13 15 15 04/30 Write block 90 len 28672 data: 11 9 9 7 7 7 8 05/30 Write block 13 len 40960 data: 15 20 17 15 15 18 19 17 16 21 06/30 Write block 442 len 8192 data: 14 14 07/30 Write block 559 len 28672 data: 13 12 12 12 11 12 7 08/30 Read block 228 len 20480 09/30 Write block 31 len 24576 data: 13 12 12 13 12 13 10/30 Write block 469 len 32768 data: 9 9 8 13 12 11 14 14 11/30 Read block 267 len 16384 12/30 Write block 128 len 8192 data: 11 10 13/30 Read block 346 len 32768 14/30 Read block 107 len 32768 15/30 Write block 152 len 12288 data: 17 18 18 16/30 Write block 248 len 8192 data: 10 11 17/30 Read block 443 len 36864 18/30 Write block 226 len 32768 data: 14 13 14 16 13 14 13 12 19/30 Flush 20/30 Write block 112 len 36864 data: 16 16 17 17 15 12 10 10 8 21/30 Read block 18 len 28672 22/30 Read block 224 len 40960 23/30 Read block 286 len 20480 24/30 Write block 588 len 8192 data: 7 9 25/30 Write block 54 len 12288 data: 9 10 11 26/30 Read block 375 len 36864 27/30 Write block 292 len 4096 data: 10 28/30 Write block 186 len 24576 data: 12 10 12 10 8 9 29/30 Write block 165 len 4096 data: 12 30/30 Read block 220 len 20480 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 10 11 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:42.515650017Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 138 138 138 227 227 227 F F F 2 040-059 136 136 136 226 226 226 T T T 3 060-079 136 136 136 226 226 226 F F F 4 080-099 138 138 138 227 227 227 F F F 5 100-119 138 138 138 227 227 227 T T T 6 120-139 138 138 138 227 227 227 T T T 7 140-159 138 138 138 227 227 227 F F F 8 160-179 136 136 136 226 226 226 T T T 9 180-199 136 136 136 226 226 226 T T T 10 200-219 136 136 136 226 226 226 F F F 11 220-239 138 138 138 227 227 227 F F F 12 240-259 138 138 138 227 227 227 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 130 130 130 214 214 214 T T T 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 138 138 138 227 227 227 F F F 23 460-479 138 138 138 227 227 227 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 135 135 135 225 225 225 F F F 27 540-559 138 138 138 227 227 227 F F F 28 560-579 138 138 138 227 227 227 F F F 29 580-599 136 136 136 226 226 226 T T T Max gen: 138, Max flush: 227 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 138 138 138 227 227 227 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 136 136 136 226 226 226 F F F 4 080-099 138 138 138 227 227 227 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 139 139 139 228 228 228 F F F 7 140-159 138 138 138 227 227 227 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 139 139 139 228 228 228 F F F 10 200-219 136 136 136 226 226 226 F F F 11 220-239 138 138 138 227 227 227 F F F 12 240-259 138 138 138 227 227 227 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 139 139 139 228 228 228 F F F 15 300-319 132 132 132 221 221 221 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 133 133 133 222 222 222 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 138 138 138 227 227 227 F F F 23 460-479 138 138 138 227 227 227 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 135 135 135 225 225 225 F F F 27 540-559 138 138 138 227 227 227 F F F 28 560-579 138 138 138 227 227 227 F F F 29 580-599 139 139 139 228 228 228 F F F Max gen: 139, Max flush: 228 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.765087409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.808470538Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809129376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809168161Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809183739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809198937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} {"msg":"Crucible ca8b7b96-d216-4bf1-9710-b6f0da05472f has session id: 6c417971-b845-44b5-9b98-c55424819c68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809771506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Upstairs opts: Upstairs UUID: ca8b7b96-d216-4bf1-9710-b6f0da05472f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809815771Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Crucible stats registered with UUID: ca8b7b96-d216-4bf1-9710-b6f0da05472f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.809843597Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:46.809858305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810325027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810351004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} {"msg":"ca8b7b96-d216-4bf1-9710-b6f0da05472f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810412357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810431304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810458091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810484617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810512694Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.81054167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810568627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810594984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.81062456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810653887Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_connection connected from Ok(127.0.0.1:64368)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810681083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_connection connected from Ok(127.0.0.1:52851)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810811377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_connection connected from Ok(127.0.0.1:53289)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.810853632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.811353569Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.811392445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.811421571Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.811449808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.81207763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812117255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812149711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812181027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812650259Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812692033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812712631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812725729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812739467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812752856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.812773993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813265562Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813286779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[0]R flush_numbers[0..12]: [227, 227, 228, 226, 227, 228, 228, 227, 228, 228, 226, 227]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813301307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[0]R generation[0..12]: [138, 138, 139, 136, 138, 139, 139, 138, 139, 139, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813315466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813329824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[1]R flush_numbers[0..12]: [227, 227, 228, 226, 227, 228, 228, 227, 228, 228, 226, 227]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813343962Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[1]R generation[0..12]: [138, 138, 139, 136, 138, 139, 139, 138, 139, 139, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813358351Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813372419Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[2]R flush_numbers[0..12]: [227, 227, 228, 226, 227, 228, 228, 227, 228, 228, 226, 227]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813386417Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[2]R generation[0..12]: [138, 138, 139, 136, 138, 139, 139, 138, 139, 139, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813400185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813414124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813427192Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.81344057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813456938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813476676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"":"downstairs","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813490284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813502462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"0","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813515991Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"1","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813529269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"client":"2","":"downstairs","session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813542737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813555216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"ca8b7b96-d216-4bf1-9710-b6f0da05472f is now active with session: 6c417971-b845-44b5-9b98-c55424819c68","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813568234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:46.813580483Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742,"session_id":"6c417971-b845-44b5-9b98-c55424819c68"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-11T23:46:46.813592971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 540 len 32768 data: 9 8 9 10 10 11 8 8 02/30 Flush 03/30 Write block 577 len 24576 data: 12 7 6 4 5 6 04/30 Write block 523 len 40960 data: 11 9 8 8 9 6 6 6 9 11 05/30 Flush 06/30 Write block 26 len 4096 data: 17 07/30 Flush 08/30 Write block 465 len 28672 data: 13 13 11 10 10 10 9 09/30 Read block 335 len 4096 10/30 Read block 218 len 32768 11/30 Write block 133 len 12288 data: 9 8 10 12/30 Write block 314 len 8192 data: 10 9 13/30 Read block 367 len 16384 14/30 Write block 451 len 32768 data: 9 12 13 12 10 11 11 8 15/30 Write block 125 len 8192 data: 8 9 16/30 Write block 219 len 28672 data: 6 10 10 11 11 11 12 17/30 Read block 481 len 16384 18/30 Write block 24 len 20480 data: 20 19 18 18 16 19/30 Write block 181 len 28672 data: 15 12 13 12 11 13 11 20/30 Read block 206 len 12288 21/30 Read block 103 len 20480 22/30 Write block 342 len 20480 data: 11 11 11 12 10 23/30 Read block 433 len 40960 24/30 Write block 541 len 4096 data: 9 25/30 Flush 26/30 Flush 27/30 Write block 562 len 8192 data: 13 12 28/30 Read block 255 len 28672 29/30 Read block 568 len 20480 30/30 Read block 518 len 12288 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:46.832877665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 136 136 136 226 226 226 F F F 4 080-099 138 138 138 227 227 227 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 138 138 138 227 227 227 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 140 140 140 232 232 232 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 138 138 138 227 227 227 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 139 139 139 228 228 228 F F F 15 300-319 140 140 140 232 232 232 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 140 140 140 232 232 232 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 140 140 140 230 230 230 F F F 27 540-559 140 140 140 232 232 232 F F F 28 560-579 140 140 140 230 230 230 T T T 29 580-599 140 140 140 230 230 230 F F F Max gen: 140, Max flush: 232 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 136 136 136 226 226 226 F F F 4 080-099 138 138 138 227 227 227 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 138 138 138 227 227 227 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 140 140 140 232 232 232 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 138 138 138 227 227 227 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 139 139 139 228 228 228 F F F 15 300-319 140 140 140 232 232 232 F F F 16 320-339 133 133 133 222 222 222 F F F 17 340-359 140 140 140 232 232 232 F F F 18 360-379 128 128 128 210 210 210 F F F 19 380-399 135 135 135 225 225 225 F F F 20 400-419 135 135 135 225 225 225 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 140 140 140 232 232 232 F F F 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 140 140 140 230 230 230 F F F 27 540-559 140 140 140 232 232 232 F F F 28 560-579 141 141 141 233 233 233 F F F 29 580-599 140 140 140 230 230 230 F F F Max gen: 141, Max flush: 233 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.050543745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.094693859Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.09540706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.095444006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.095458444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.095473262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} {"msg":"Crucible beb6c138-dbc0-49ed-9f7e-28b5b13b9c1f has session id: 1fa05731-47cf-47ed-ba94-9786f60a30f6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.095937904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Upstairs opts: Upstairs UUID: beb6c138-dbc0-49ed-9f7e-28b5b13b9c1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.095980719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Crucible stats registered with UUID: beb6c138-dbc0-49ed-9f7e-28b5b13b9c1f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096006126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:51.096026883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096501084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096542029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} {"msg":"beb6c138-dbc0-49ed-9f7e-28b5b13b9c1f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096598342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096635448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096677732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096708959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096738745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096767581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096794638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096820325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096846191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096873488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63359)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096901295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36332)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096929941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59896)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.096961857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.097491771Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.097531156Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.097563802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.097590999Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.09806319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098203353Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098677664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.0987058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098721098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098805158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098824876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098838794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.098859001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.09887305Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099418462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099902582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099924969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[0]R flush_numbers[0..12]: [227, 232, 228, 226, 227, 228, 232, 227, 228, 232, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099945266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[0]R generation[0..12]: [138, 140, 139, 136, 138, 139, 140, 138, 139, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099960595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099975133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[1]R flush_numbers[0..12]: [227, 232, 228, 226, 227, 228, 232, 227, 228, 232, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.099989241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[1]R generation[0..12]: [138, 140, 139, 136, 138, 139, 140, 138, 139, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100003209Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100017377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[2]R flush_numbers[0..12]: [227, 232, 228, 226, 227, 228, 232, 227, 228, 232, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100031166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[2]R generation[0..12]: [138, 140, 139, 136, 138, 139, 140, 138, 139, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100044934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100059252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.10007267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100086159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100099367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100112745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"":"downstairs","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100126154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,Wait for a query_work_queue command to finish before sending IO "session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100148861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"0","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100162489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"1","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100175658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"client":"2","":"downstairs","session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100188986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100201554Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"beb6c138-dbc0-49ed-9f7e-28b5b13b9c1f is now active with session: 1fa05731-47cf-47ed-ba94-9786f60a30f6","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100214473Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.100226581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750,"session_id":"1fa05731-47cf-47ed-ba94-9786f60a30f6"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:51.10023904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 12288 data: 9 8 6 02/30 Read block 99 len 8192 03/30 Write block 532 len 28672 data: 12 12 11 13 13 10 10 04/30 Read block 291 len 12288 05/30 Read block 243 len 36864 06/30 Read block 441 len 24576 07/30 Write block 73 len 36864 data: 12 11 11 9 9 8 9 9 8 08/30 Write block 234 len 36864 data: 12 12 12 8 9 9 9 9 9 09/30 Read block 451 len 8192 10/30 Read block 262 len 20480 11/30 Write block 547 len 32768 data: 9 5 7 7 9 10 9 8 12/30 Write block 284 len 12288 data: 10 13 9 13/30 Read block 361 len 8192 14/30 Write block 394 len 40960 data: 11 16 16 15 15 17 14 13 13 11 15/30 Read block 424 len 12288 16/30 Write block 573 len 36864 data: 10 9 9 11 13 8 7 5 6 17/30 Write block 455 len 16384 data: 11 12 12 9 18/30 Read block 436 len 40960 19/30 Flush 20/30 Write block 378 len 24576 data: 10 9 9 9 7 8 21/30 Write block 330 len 8192 data: 9 8 22/30 Read block 427 len 8192 23/30 Write block 72 len 20480 data: 12 13 12 12 10 24/30 Write block 291 len 24576 data: 11 11 9 9 12 14 25/30 Read block 0 len 16384 26/30 Write block 307 len 20480 data: 7 7 8 9 9 27/30 Write block 313 len 16384 data: 14 11 10 11 28/30 Write block 586 len 16384 data: 6 7 8 10 29/30 Write block 584 len 4096 data: 7 30/30 Write block 552 len 36864 data: 11 10 9 7 7 10 11 14 13 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:51.119727988Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 142 142 142 234 234 234 T T T 4 080-099 142 142 142 234 234 234 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 138 138 138 227 227 227 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 140 140 140 232 232 232 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 142 142 142 234 234 234 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 142 142 142 234 234 234 T T T 15 300-319 140 140 140 232 232 232 T T T 16 320-339 133 133 133 222 222 222 T T T 17 340-359 140 140 140 232 232 232 F F F 18 360-379 128 128 128 210 210 210 T T T 19 380-399 142 142 142 234 234 234 T T T 20 400-419 142 142 142 234 234 234 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 142 142 142 234 234 234 F F F 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 142 142 142 234 234 234 T T T 28 560-579 142 142 142 234 234 234 T T T 29 580-599 142 142 142 234 234 234 T T T Max gen: 142, Max flush: 234 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 143 143 143 235 235 235 F F F 4 080-099 142 142 142 234 234 234 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 140 140 140 232 232 232 F F F 7 140-159 138 138 138 227 227 227 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 140 140 140 232 232 232 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 142 142 142 234 234 234 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 143 143 143 235 235 235 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 143 143 143 235 235 235 F F F 17 340-359 140 140 140 232 232 232 F F F 18 360-379 143 143 143 235 235 235 F F F 19 380-399 143 143 143 235 235 235 F F F 20 400-419 142 142 142 234 234 234 F F F 21 420-439 125 125 125 207 207 207 F F F 22 440-459 142 142 142 234 234 234 F F F 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 143 143 143 235 235 235 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 143 143 143 235 235 235 F F F Max gen: 143, Max flush: 235 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.384413456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.428307131Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429239135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.42928251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429298028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429313626Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} {"msg":"Crucible 6eb37948-ed31-463b-b5c0-bc05eb51f9f4 has session id: a0f5774d-e181-4d68-8c17-856822d19d24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429865258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Upstairs opts: Upstairs UUID: 6eb37948-ed31-463b-b5c0-bc05eb51f9f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429914052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Crucible stats registered with UUID: 6eb37948-ed31-463b-b5c0-bc05eb51f9f4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.429942048Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:55.429964245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430471712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430499619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} {"msg":"6eb37948-ed31-463b-b5c0-bc05eb51f9f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430562031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430594257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430626213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430645321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430662139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430677546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430693015Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430706573Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430721661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430736239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:52384)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430848035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:53057)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43089344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_connection connected from Ok(127.0.0.1:47799)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.430925506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.431770441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.431801517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.431954488Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.431987084Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43202065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.432037297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.432543095Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.432574701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43265889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433178236Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433206892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43322258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433290432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43330857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433322438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433334846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433348245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[0]R flush_numbers[0..12]: [227, 232, 228, 235, 234, 228, 232, 227, 228, 232, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433362313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[0]R generation[0..12]: [138, 140, 139, 143, 142, 139, 140, 138, 139, 140, 140, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433380661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433395409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[1]R flush_numbers[0..12]: [227, 232, 228, 235, 234, 228, 232, 227, 228, 232, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433409717Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[1]R generation[0..12]: [138, 140, 139, 143, 142, 139, 140, 138, 139, 140, 140, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433423885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433438054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[2]R flush_numbers[0..12]: [227, 232, 228, 235, 234, 228, 232, 227, 228, 232, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433452172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[2]R generation[0..12]: [138, 140, 139, 143, 142, 139, 140, 138, 139, 140, 140, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43346605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433480108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433493437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433506995Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433520303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433533482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"":"downstairs","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43354709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433559348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"0","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-12-11T23:46:55.433585605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"1","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433603813Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"client":"2","":"downstairs","session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433617701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.43363018Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"6eb37948-ed31-463b-b5c0-bc05eb51f9f4 is now active with session: a0f5774d-e181-4d68-8c17-856822d19d24","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433643028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433655307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758,"session_id":"a0f5774d-e181-4d68-8c17-856822d19d24"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:55.433667735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 16384 data: 8 9 11 11 02/30 Read block 72 len 28672 03/30 Read block 364 len 24576 04/30 Read block 388 len 32768 05/30 Read block 582 len 4096 06/30 Write block 84 len 28672 data: 13 13 15 13 15 15 12 07/30 Flush 08/30 Write block 145 len 4096 data: 12 09/30 Read block 365 len 12288 10/30 Read block 577 len 28672 11/30 Flush 12/30 Write block 426 len 8192 data: 15 13 13/30 Write block 441 len 28672 data: 12 15 15 14 11 12 14 14/30 Flush 15/30 Write block 122 len 36864 data: 10 8 9 9 10 11 12 11 12 16/30 Read block 15 len 24576 17/30 Read block 16 len 20480 18/30 Read block 308 len 40960 19/30 Write block 248 len 28672 data: 11 12 13 7 7 10 8 20/30 Read block 451 len 16384 21/30 Flush 22/30 Write block 366 len 36864 data: 13 13 11 13 13 13 12 9 7 23/30 Write block 438 len 28672 data: 13 16 14 13 16 16 15 24/30 Write block 505 len 40960 data: 13 14 14 14 13 14 12 13 13 10 25/30 Read block 42 len 4096 26/30 Write block 127 len 28672 data: 12 13 12 13 9 9 10 27/30 Write block 287 len 40960 data: 11 15 12 10 12 12 10 10 13 15 28/30 Read block 403 len 4096 29/30 Read block 390 len 40960 30/30 Write block 208 len 16384 data: 10 11 13 16 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:55.45470764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 143 143 143 235 235 235 F F F 4 080-099 144 144 144 236 236 236 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 144 144 144 239 239 239 T T T 7 140-159 144 144 144 237 237 237 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 140 140 140 232 232 232 T T T 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 143 143 143 235 235 235 T T T 15 300-319 143 143 143 235 235 235 F F F 16 320-339 143 143 143 235 235 235 F F F 17 340-359 140 140 140 232 232 232 F F F 18 360-379 143 143 143 235 235 235 T T T 19 380-399 143 143 143 235 235 235 F F F 20 400-419 142 142 142 234 234 234 F F F 21 420-439 144 144 144 238 238 238 T T T 22 440-459 144 144 144 238 238 238 T T T 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 135 135 135 225 225 225 T T T 26 520-539 142 142 142 234 234 234 F F F 27 540-559 143 143 143 235 235 235 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 144 144 144 236 236 236 F F F Max gen: 144, Max flush: 239 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 143 143 143 235 235 235 F F F 4 080-099 144 144 144 236 236 236 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 145 145 145 240 240 240 F F F 7 140-159 144 144 144 237 237 237 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 F F F 10 200-219 145 145 145 240 240 240 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 143 143 143 235 235 235 F F F 17 340-359 140 140 140 232 232 232 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 143 143 143 235 235 235 F F F 20 400-419 142 142 142 234 234 234 F F F 21 420-439 145 145 145 240 240 240 F F F 22 440-459 145 145 145 240 240 240 F F F 23 460-479 140 140 140 232 232 232 F F F 24 480-499 132 132 132 219 219 219 F F F 25 500-519 145 145 145 240 240 240 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 143 143 143 235 235 235 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 144 144 144 236 236 236 F F F Max gen: 145, Max flush: 240 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.707153359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:59.749525883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.749941292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.781314902Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782190074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782224509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782238388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782254116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} {"msg":"Crucible a3fe24cd-d66d-48f4-ae14-49026657bd7b has session id: 3d7df6f1-72d6-41b5-9831-b71ee80678d0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782769012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Upstairs opts: Upstairs UUID: a3fe24cd-d66d-48f4-ae14-49026657bd7b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782806207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Crucible stats registered with UUID: a3fe24cd-d66d-48f4-ae14-49026657bd7b","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.782823695Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:59.782845082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783353539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783380626Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} {"msg":"a3fe24cd-d66d-48f4-ae14-49026657bd7b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783428Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783446298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783468835Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783483773Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783499271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783521188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783536826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783555224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783580911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783607027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35540)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783767208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52627)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783810132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61523)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.783857476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.784669326Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.784698352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.78471418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.784732928Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.784813278Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.784835865Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.785319725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.785350041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.785489884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.785934518Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.785974243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.78600134Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786016228Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786031256Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786044765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786122365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786142283Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[0]R flush_numbers[0..12]: [227, 232, 228, 235, 236, 228, 240, 237, 228, 232, 240, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786157561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[0]R generation[0..12]: [138, 140, 139, 143, 144, 139, 145, 144, 139, 140, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786176078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786190677Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[1]R flush_numbers[0..12]: [227, 232, 228, 235, 236, 228, 240, 237, 228, 232, 240, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786204725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[1]R generation[0..12]: [138, 140, 139, 143, 144, 139, 145, 144, 139, 140, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786218613Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786233811Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[2]R flush_numbers[0..12]: [227, 232, 228, 235, 236, 228, 240, 237, 228, 232, 240, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786248739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[2]R generation[0..12]: [138, 140, 139, 143, 144, 139, 145, 144, 139, 140, 145, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786262608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786288964Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786315441Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786342678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786357936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"":"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786378953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786411049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786436906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"0","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786459233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"1","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786474041Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"client":"2","":"downstairs","session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786495828Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786520505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"a3fe24cd-d66d-48f4-ae14-49026657bd7b is now active with session: 3d7df6f1-72d6-41b5-9831-b71ee80678d0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786546022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.78656343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766,"session_id":"3d7df6f1-72d6-41b5-9831-b71ee80678d0"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:46:59.786577658Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 65 len 28672 data: 13 10 10 13 9 11 13 02/30 Write block 202 len 40960 data: 12 9 9 10 9 9 11 12 14 17 03/30 Read block 308 len 16384 04/30 Write block 488 len 36864 data: 10 11 11 7 9 11 11 15 16 05/30 Write block 398 len 16384 data: 16 18 15 14 06/30 Read block 497 len 36864 07/30 Write block 122 len 32768 data: 11 9 10 10 11 13 14 13 08/30 Read block 19 len 28672 09/30 Read block 272 len 24576 10/30 Read block 38 len 8192 11/30 Read block 511 len 4096 12/30 Flush 13/30 Write block 198 len 36864 data: 13 12 13 12 13 10 10 11 10 14/30 Write block 506 len 12288 data: 15 15 15 15/30 Read block 92 len 32768 16/30 Read block 535 len 32768 17/30 Write block 193 len 40960 data: 12 14 12 11 10 14 13 14 13 14 18/30 Read block 311 len 8192 19/30 Read block 69 len 8192 20/30 Write block 357 len 12288 data: 7 8 10 21/30 Read block 494 len 28672 22/30 Write block 443 len 36864 data: 17 16 12 13 15 9 9 12 10 23/30 Write block 544 len 12288 data: 11 12 9 24/30 Read block 439 len 32768 25/30 Write block 409 len 12288 data: 10 9 9 26/30 Write block 32 len 32768 data: 13 13 14 13 14 11 12 14 27/30 Write block 441 len 32768 data: 14 17 18 17 13 14 16 10 28/30 Read block 344 len 40960 29/30 Read block 104 len 40960 30/30 Write block 478 len 12288 data: 16 16 16 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:46:59.807855124Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 140 140 140 232 232 232 T T T 2 040-059 139 139 139 228 228 228 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 144 144 144 236 236 236 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 146 146 146 241 241 241 F F F 7 140-159 144 144 144 237 237 237 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 140 140 140 232 232 232 T T T 10 200-219 146 146 146 241 241 241 T T T 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 143 143 143 235 235 235 F F F 17 340-359 140 140 140 232 232 232 T T T 18 360-379 145 145 145 240 240 240 F F F 19 380-399 146 146 146 241 241 241 F F F 20 400-419 146 146 146 241 241 241 T T T 21 420-439 145 145 145 240 240 240 F F F 22 440-459 145 145 145 240 240 240 T T T 23 460-479 140 140 140 232 232 232 T T T 24 480-499 146 146 146 241 241 241 T T T 25 500-519 145 145 145 240 240 240 T T T 26 520-539 142 142 142 234 234 234 F F F 27 540-559 143 143 143 235 235 235 T T T 28 560-579 143 143 143 235 235 235 F F F 29 580-599 144 144 144 236 236 236 F F F Max gen: 146, Max flush: 241 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 227 227 227 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 139 139 139 228 228 228 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 144 144 144 236 236 236 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 146 146 146 241 241 241 F F F 7 140-159 144 144 144 237 237 237 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 143 143 143 235 235 235 F F F 17 340-359 147 147 147 242 242 242 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 146 146 146 241 241 241 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 145 145 145 240 240 240 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 147 147 147 242 242 242 F F F 25 500-519 147 147 147 242 242 242 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 147 147 147 242 242 242 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 144 144 144 236 236 236 F F F Max gen: 147, Max flush: 242 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.111681888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.155365999Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156046175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.15608383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156099348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156115416Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} {"msg":"Crucible e4b3104e-3bf8-4a10-9a8f-8e91d62b0a57 has session id: 84b29405-27d1-44b1-b5a8-9b1ba84febd2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156585948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Upstairs opts: Upstairs UUID: e4b3104e-3bf8-4a10-9a8f-8e91d62b0a57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156628672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Crucible stats registered with UUID: e4b3104e-3bf8-4a10-9a8f-8e91d62b0a57","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.156654189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:04.156676426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157167755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157193832Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} {"msg":"e4b3104e-3bf8-4a10-9a8f-8e91d62b0a57 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157266013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157285891Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157308808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157323946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157338384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157353442Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157367911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157382099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157399607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157427343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42378)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.157554087Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61563)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.15761298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:04.157651155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:04.157738604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:04.157918822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.158034988Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.158432898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.158459035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.158474263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.158488261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159116683Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159261855Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159630759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159669944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159699791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159817746Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159838273Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:04.159852512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.158996818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55582)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.159160218Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.160130397Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.160157384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.160643193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161185796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161209023Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[0]R flush_numbers[0..12]: [227, 242, 228, 241, 236, 228, 241, 237, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161225391Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[0]R generation[0..12]: [138, 147, 139, 146, 144, 139, 146, 144, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161240109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161255037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[1]R flush_numbers[0..12]: [227, 242, 228, 241, 236, 228, 241, 237, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161269276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[1]R generation[0..12]: [138, 147, 139, 146, 144, 139, 146, 144, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161283474Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161297872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[2]R flush_numbers[0..12]: [227, 242, 228, 241, 236, 228, 241, 237, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.16131205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[2]R generation[0..12]: [138, 147, 139, 146, 144, 139, 146, 144, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161326319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161340737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161357925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161372383Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161386031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161399399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"":"downstairs","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161413188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161425636Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"0","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161439345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"1","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161452903Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"client":"2","":"downstairs","session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161466661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.16147927Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"e4b3104e-3bf8-4a10-9a8f-8e91d62b0a57 is now active with session: 84b29405-27d1-44b1-b5a8-9b1ba84febd2","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:47:14.161492528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161511286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774,"session_id":"84b29405-27d1-44b1-b5a8-9b1ba84febd2"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:14.161524304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 20480 data: 12 12 14 14 14 02/30 Read block 54 len 4096 03/30 Read block 110 len 36864 04/30 Read block 590 len 12288 05/30 Flush 06/30 Write block 142 len 40960 data: 8 8 9 13 11 9 11 13 12 12 07/30 Write block 320 len 32768 data: 14 12 14 14 13 13 15 15 08/30 Read block 483 len 40960 09/30 Read block 507 len 8192 10/30 Write block 8 len 36864 data: 17 16 15 15 16 16 21 18 16 11/30 Read block 103 len 36864 12/30 Read block 345 len 28672 13/30 Write block 580 len 4096 data: 6 14/30 Read block 59 len 4096 15/30 Flush 16/30 Write block 89 len 4096 data: 16 17/30 Write block 422 len 20480 data: 12 15 16 16 16 18/30 Write block 15 len 12288 data: 19 17 16 19/30 Read block 268 len 20480 20/30 Read block 259 len 4096 21/30 Read block 199 len 32768 22/30 Read block 420 len 12288 23/30 Read block 416 len 16384 24/30 Flush 25/30 Write block 57 len 20480 data: 11 12 16 13 16 26/30 Read block 572 len 12288 27/30 Read block 373 len 4096 28/30 Read block 434 len 4096 29/30 Read block 340 len 36864 30/30 Read block 305 len 24576 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:14.1817547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 139 139 139 228 228 228 T T T 3 060-079 146 146 146 241 241 241 T T T 4 080-099 148 148 148 245 245 245 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 146 146 146 241 241 241 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 147 147 147 242 242 242 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 146 146 146 241 241 241 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 147 147 147 242 242 242 F F F 25 500-519 147 147 147 242 242 242 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 147 147 147 242 242 242 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 148 148 148 244 244 244 F F F Max gen: 148, Max flush: 245 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 149 149 149 246 246 246 F F F 3 060-079 149 149 149 246 246 246 F F F 4 080-099 148 148 148 245 245 245 F F F 5 100-119 139 139 139 228 228 228 F F F 6 120-139 146 146 146 241 241 241 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 142 142 142 234 234 234 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 136 136 136 226 226 226 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 147 147 147 242 242 242 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 146 146 146 241 241 241 F F F 20 400-419 147 147 147 242 242 242 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 147 147 147 242 242 242 F F F 25 500-519 147 147 147 242 242 242 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 147 147 147 242 242 242 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 148 148 148 244 244 244 F F F Max gen: 149, Max flush: 246 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.404920288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.448279979Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.448978702Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.449017728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.449033166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.449048504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} {"msg":"Crucible c09c800a-a152-4434-b3b5-b25a24ae9f65 has session id: 54347fbf-3dc0-4b2f-865f-a55c29a0eba9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.44955846Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Upstairs opts: Upstairs UUID: c09c800a-a152-4434-b3b5-b25a24ae9f65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.449603605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Crucible stats registered with UUID: c09c800a-a152-4434-b3b5-b25a24ae9f65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.449629341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:18.449658028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.45004612Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450085065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} {"msg":"c09c800a-a152-4434-b3b5-b25a24ae9f65 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450143188Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450176373Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.45020512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450225947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450243485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450258293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450272492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.45028657Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450308057Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450326235Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53373)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450342313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45355)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450417874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:18.450548647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:18.450597731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:18.450632517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.450737574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.451097949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.451125436Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.451141753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.451161081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.451781604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.45189607Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.45229772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.452339165Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.452368171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.452396108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.452425274Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:18.452450101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.451677147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59831)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.451837877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.452966217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.452997453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.45350285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.45406485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454086987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[0]R flush_numbers[0..12]: [245, 242, 246, 246, 245, 228, 241, 244, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454115674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[0]R generation[0..12]: [148, 147, 149, 149, 148, 139, 146, 148, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454132022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454149059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[1]R flush_numbers[0..12]: [245, 242, 246, 246, 245, 228, 241, 244, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454165117Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[1]R generation[0..12]: [148, 147, 149, 149, 148, 139, 146, 148, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454180726Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454196634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[2]R flush_numbers[0..12]: [245, 242, 246, 246, 245, 228, 241, 244, 228, 242, 242, 234]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454212262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[2]R generation[0..12]: [148, 147, 149, 149, 148, 139, 146, 148, 139, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454233239Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454249777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454264505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454279363Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454293921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.45430857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"":"downstairs","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454324858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454347405Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"0","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454380121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"1","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454405328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"client":"2","":"downstairs","session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454427295Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454452412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"c09c800a-a152-4434-b3b5-b25a24ae9f65 is now active with session: 54347fbf-3dc0-4b2f-865f-a55c29a0eba9","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454481778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454508345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782,"session_id":"54347fbf-3dc0-4b2f-865f-a55c29a0eba9"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:28.454525563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 495 len 40960 data: 16 17 15 13 15 13 10 12 14 14 02/30 Write block 57 len 28672 data: 12 13 17 14 17 16 12 03/30 Write block 262 len 36864 data: 11 11 13 13 11 11 9 8 5 04/30 Read block 267 len 32768 05/30 Read block 360 len 20480 06/30 Write block 400 len 20480 data: 16 15 14 12 11 07/30 Read block 296 len 36864 08/30 Write block 287 len 12288 data: 12 16 13 09/30 Read block 263 len 32768 10/30 Write block 554 len 12288 data: 10 8 8 11/30 Read block 15 len 40960 12/30 Write block 79 len 36864 data: 10 10 9 10 11 14 14 16 14 13/30 Flush 14/30 Write block 597 len 8192 data: 6 5 15/30 Write block 114 len 24576 data: 18 18 16 13 11 11 16/30 Write block 350 len 28672 data: 8 8 8 8 7 10 10 17/30 Read block 406 len 32768 18/30 Read block 266 len 32768 19/30 Read block 136 len 24576 20/30 Read block 92 len 36864 21/30 Write block 540 len 12288 data: 10 10 10 22/30 Flush 23/30 Write block 282 len 12288 data: 12 11 11 24/30 Read block 16 len 28672 25/30 Read block 133 len 4096 26/30 Flush 27/30 Write block 224 len 16384 data: 12 13 15 14 28/30 Write block 124 len 8192 data: 11 11 29/30 Write block 360 len 4096 data: 10 30/30 Read block 257 len 28672 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 3 5 Done 2 4 1 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:28.474460895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 150 150 150 247 247 247 F F F 5 100-119 150 150 150 248 248 248 F F F 6 120-139 146 146 146 241 241 241 T T T 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 142 142 142 234 234 234 T T T 12 240-259 144 144 144 239 239 239 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 150 150 150 248 248 248 F F F 18 360-379 145 145 145 240 240 240 T T T 19 380-399 146 146 146 241 241 241 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 150 150 150 247 247 247 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 150 150 150 248 248 248 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 150 150 150 248 248 248 F F F Max gen: 150, Max flush: 249 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 150 150 150 247 247 247 F F F 5 100-119 150 150 150 248 248 248 F F F 6 120-139 151 151 151 250 250 250 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 151 151 151 250 250 250 F F F 12 240-259 144 144 144 239 239 239 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 249 249 249 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 150 150 150 248 248 248 F F F 18 360-379 151 151 151 250 250 250 F F F 19 380-399 146 146 146 241 241 241 F F F 20 400-419 150 150 150 247 247 247 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 150 150 150 247 247 247 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 150 150 150 248 248 248 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 150 150 150 248 248 248 F F F Max gen: 151, Max flush: 250 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.704600106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.748332361Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.749317529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.749357154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.749372092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.74938797Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} {"msg":"Crucible 05d30a8e-2893-488f-88b7-72e4c67588e4 has session id: 06f2f957-56ba-46f3-ac73-c35c6a0d0251","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.749980257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Upstairs opts: Upstairs UUID: 05d30a8e-2893-488f-88b7-72e4c67588e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750027461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Crucible stats registered with UUID: 05d30a8e-2893-488f-88b7-72e4c67588e4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750053638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:32.750079804Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750605439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750670401Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} {"msg":"05d30a8e-2893-488f-88b7-72e4c67588e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750697418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750719395Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750751171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750811673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75083547Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750860827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750885784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750909111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750937118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.750968464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_connection connected from Ok(127.0.0.1:47184)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751005729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_connection connected from Ok(127.0.0.1:53226)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751180638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_connection connected from Ok(127.0.0.1:40997)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751210054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751574179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751634151Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751683735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.751715751Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75236678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.752402566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.752800046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.752840661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753037077Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753084721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753110328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753123996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753138034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753157172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753381544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753859245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753880672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[0]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 248, 250, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75389936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[0]R generation[0..12]: [148, 147, 150, 150, 150, 150, 151, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753914258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753928676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[1]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 248, 250, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753942884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[1]R generation[0..12]: [148, 147, 150, 150, 150, 150, 151, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753956863Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753970941Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[2]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 248, 250, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753984789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[2]R generation[0..12]: [148, 147, 150, 150, 150, 150, 151, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.753999647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754013895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754030764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754044712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75405786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754100115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"":"downstairs","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754150149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754177225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"0","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754194353Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"1","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75421898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"client":"2","":"downstairs","session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754246517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754271754Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"05d30a8e-2893-488f-88b7-72e4c67588e4 is now active with session: 06f2f957-56ba-46f3-ac73-c35c6a0d0251","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754286992Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.75430037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790,"session_id":"06f2f957-56ba-46f3-ac73-c35c6a0d0251"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:32.754323047Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 110 len 4096 data: 10 02/30 Read block 273 len 12288 03/30 Write block 338 len 8192 data: 14 16 04/30 Read block 0 len 4096 05/30 Read block 54 len 8192 06/30 Write block 128 len 24576 data: 15 14 14 10 10 11 07/30 Read block 128 len 32768 08/30 Read block 566 len 28672 09/30 Write block 550 len 40960 data: 8 10 12 11 11 9 9 11 12 15 10/30 Read block 75 len 36864 11/30 Write block 486 len 32768 data: 10 11 11 12 12 8 10 12 12/30 Write block 482 len 12288 data: 16 16 12 13/30 Read block 75 len 28672 14/30 Write block 349 len 32768 data: 8 9 9 9 9 8 11 11 15/30 Write block 290 len 32768 data: 11 13 13 11 11 14 16 13 16/30 Read block 226 len 8192 17/30 Flush 18/30 Write block 398 len 36864 data: 17 19 17 16 15 13 12 9 10 19/30 Read block 514 len 20480 20/30 Write block 324 len 32768 data: 14 14 16 16 15 11 10 9 21/30 Read block 310 len 8192 22/30 Read block 114 len 16384 23/30 Read block 392 len 40960 24/30 Read block 364 len 8192 25/30 Write block 117 len 20480 data: 14 12 12 9 9 26/30 Flush 27/30 Write block 341 len 40960 data: 13 12 12 12 13 11 9 9 9 10 28/30 Write block 250 len 28672 data: 14 8 8 11 9 6 9 29/30 Read block 539 len 4096 30/30 Read block 203 len 24576 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:32.775247766Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 150 150 150 247 247 247 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 151 151 151 250 250 250 F F F 12 240-259 144 144 144 239 239 239 T T T 13 260-279 150 150 150 247 247 247 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 152 152 152 251 251 251 T T T 18 360-379 151 151 151 250 250 250 F F F 19 380-399 152 152 152 252 252 252 F F F 20 400-419 152 152 152 252 252 252 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 152 152 152 251 251 251 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 150 150 150 248 248 248 F F F Max gen: 152, Max flush: 252 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 147 147 147 242 242 242 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 150 150 150 247 247 247 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 F F F 9 180-199 147 147 147 242 242 242 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 151 151 151 250 250 250 F F F 12 240-259 153 153 153 253 253 253 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 143 143 143 235 235 235 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 151 151 151 250 250 250 F F F 19 380-399 152 152 152 252 252 252 F F F 20 400-419 152 152 152 252 252 252 F F F 21 420-439 148 148 148 245 245 245 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 147 147 147 242 242 242 F F F 24 480-499 152 152 152 251 251 251 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 142 142 142 234 234 234 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 143 143 143 235 235 235 F F F 29 580-599 150 150 150 248 248 248 F F F Max gen: 153, Max flush: 253 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.001188608Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:37.044067309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.044475638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.07576961Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.07649249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.076530745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.076545843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.076562771Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} {"msg":"Crucible 805373ce-ab92-4fc4-95b0-ab2154f289c2 has session id: be527bb8-c640-4ab0-b5bd-1334345d0fa2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077068478Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Upstairs opts: Upstairs UUID: 805373ce-ab92-4fc4-95b0-ab2154f289c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077119352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Crucible stats registered with UUID: 805373ce-ab92-4fc4-95b0-ab2154f289c2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077145649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:37.077161627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077609971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077635958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} {"msg":"805373ce-ab92-4fc4-95b0-ab2154f289c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.07769998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077718718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077742515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077757742Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077772421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077786619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077809116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077834843Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.07786138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077884207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60940)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077910994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59347)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077946169Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43795)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.077977195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.07874089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.078779646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.078811542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.078842288Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.078871744Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.078900111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.079459101Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.079489198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.079507175Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080073005Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.08011374Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080144716Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080169563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080197849Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080222556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080244973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080269481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[0]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 252, 252, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080294587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[0]R generation[0..12]: [148, 147, 150, 150, 150, 152, 152, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080318974Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080345851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[1]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 252, 252, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080373748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[1]R generation[0..12]: [148, 147, 150, 150, 150, 152, 152, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080399474Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080428651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[2]R flush_numbers[0..12]: [245, 242, 247, 247, 247, 252, 252, 244, 228, 242, 242, 250]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:47:37.080458647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[2]R generation[0..12]: [148, 147, 150, 150, 150, 152, 152, 148, 139, 147, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080492263Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080522349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080550775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080584181Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080614738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080643354Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"":"downstairs","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-11T23:47:37.0806711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080696467Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"0","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080712715Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"1","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080726424Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"client":"2","":"downstairs","session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080739942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.08075354Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"805373ce-ab92-4fc4-95b0-ab2154f289c2 is now active with session: be527bb8-c640-4ab0-b5bd-1334345d0fa2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080766759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080778997Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798,"session_id":"be527bb8-c640-4ab0-b5bd-1334345d0fa2"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:37.080791545Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 8192 data: 9 10 02/30 Read block 381 len 20480 03/30 Read block 48 len 8192 04/30 Write block 225 len 24576 data: 14 16 15 15 17 14 05/30 Read block 397 len 16384 06/30 Read block 74 len 36864 07/30 Write block 119 len 4096 data: 13 08/30 Read block 250 len 24576 09/30 Write block 92 len 28672 data: 10 8 8 8 9 9 7 10/30 Read block 77 len 4096 11/30 Write block 23 len 24576 data: 22 21 20 19 19 17 12/30 Read block 289 len 32768 13/30 Write block 390 len 20480 data: 9 11 9 10 12 14/30 Flush 15/30 Write block 465 len 16384 data: 14 14 12 11 16/30 Read block 340 len 24576 17/30 Flush 18/30 Write block 255 len 24576 data: 7 10 10 9 7 7 19/30 Read block 255 len 20480 20/30 Read block 111 len 4096 21/30 Write block 571 len 40960 data: 10 9 11 10 10 12 14 9 8 7 22/30 Write block 302 len 16384 data: 10 10 9 8 23/30 Write block 424 len 28672 data: 17 17 17 14 16 13 8 24/30 Write block 256 len 16384 data: 11 11 10 8 25/30 Read block 161 len 8192 26/30 Flush 27/30 Write block 176 len 28672 data: 15 14 17 16 16 16 13 28/30 Read block 20 len 12288 29/30 Write block 531 len 32768 data: 10 13 13 12 14 14 11 11 30/30 Read block 429 len 24576 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:37.100560599Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 154 154 154 254 254 254 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 139 139 139 228 228 228 T T T 9 180-199 147 147 147 242 242 242 T T T 10 200-219 147 147 147 242 242 242 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 151 151 151 250 250 250 F F F 19 380-399 154 154 154 254 254 254 F F F 20 400-419 152 152 152 252 252 252 F F F 21 420-439 154 154 154 256 256 256 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 154 154 154 255 255 255 F F F 24 480-499 152 152 152 251 251 251 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 142 142 142 234 234 234 T T T 27 540-559 152 152 152 251 251 251 F F F 28 560-579 154 154 154 256 256 256 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 154, Max flush: 256 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 150 150 150 247 247 247 F F F 3 060-079 150 150 150 247 247 247 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 154 154 154 254 254 254 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 148 148 148 244 244 244 F F F 8 160-179 155 155 155 257 257 257 F F F 9 180-199 155 155 155 257 257 257 F F F 10 200-219 147 147 147 242 242 242 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 151 151 151 250 250 250 F F F 19 380-399 154 154 154 254 254 254 F F F 20 400-419 152 152 152 252 252 252 F F F 21 420-439 154 154 154 256 256 256 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 154 154 154 255 255 255 F F F 24 480-499 152 152 152 251 251 251 F F F 25 500-519 150 150 150 247 247 247 F F F 26 520-539 155 155 155 257 257 257 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 154 154 154 256 256 256 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 155, Max flush: 257 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.367612072Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.412200532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.412955458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.412994453Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.41301722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.413038307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} {"msg":"Crucible eba06e99-fb03-4f20-bf01-39646d8c17c4 has session id: aac08a23-c512-4ae5-92a5-22e8aeb168f2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.413465154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Upstairs opts: Upstairs UUID: eba06e99-fb03-4f20-bf01-39646d8c17c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.413508459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Crucible stats registered with UUID: eba06e99-fb03-4f20-bf01-39646d8c17c4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.413534566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:41.413556573Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.413985739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414023535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} {"msg":"eba06e99-fb03-4f20-bf01-39646d8c17c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414081598Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414119763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414158118Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414185555Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414208912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414235179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414321558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414369122Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414400768Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414426895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:32931)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414450632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65203)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414473139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60371)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.414500855Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415043248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415076224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415092082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415707006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415849408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415869936Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.415884584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416345296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416372553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416387761Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416468841Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416489858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416503497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.416576078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.41704433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417063507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[0]R flush_numbers[0..12]: [245, 254, 247, 247, 254, 254, 252, 244, 257, 257, 242, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417082465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[0]R generation[0..12]: [148, 154, 150, 150, 154, 154, 152, 148, 155, 155, 147, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417097003Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417111171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[1]R flush_numbers[0..12]: [245, 254, 247, 247, 254, 254, 252, 244, 257, 257, 242, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417125289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[1]R generation[0..12]: [148, 154, 150, 150, 154, 154, 152, 148, 155, 155, 147, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417139158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417153466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[2]R flush_numbers[0..12]: [245, 254, 247, 247, 254, 254, 252, 244, 257, 257, 242, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417167494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[2]R generation[0..12]: [148, 154, 150, 150, 154, 154, 152, 148, 155, 155, 147, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417181533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417195831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417208919Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417222347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417235526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417248484Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"":"downstairs","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417261732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417276591Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"0","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417297718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"1","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417311196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"client":"2","":"downstairs","session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417324525Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417337093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"eba06e99-fb03-4f20-bf01-39646d8c17c4 is now active with session: aac08a23-c512-4ae5-92a5-22e8aeb168f2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417350012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.41736212Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806,"session_id":"aac08a23-c512-4ae5-92a5-22e8aeb168f2"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:41.417374558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 36864 data: 15 18 17 17 17 14 14 13 12 02/30 Write block 161 len 16384 data: 10 8 9 13 03/30 Write block 568 len 20480 data: 8 6 8 11 10 04/30 Read block 65 len 16384 05/30 Read block 96 len 40960 06/30 Read block 128 len 4096 07/30 Write block 508 len 32768 data: 16 14 15 13 14 14 11 10 08/30 Read block 317 len 20480 09/30 Read block 369 len 24576 10/30 Write block 58 len 20480 data: 14 18 15 18 17 11/30 Write block 203 len 36864 data: 11 11 12 11 10 12 13 15 18 12/30 Write block 403 len 24576 data: 14 13 10 11 9 9 13/30 Read block 68 len 4096 14/30 Write block 460 len 12288 data: 14 11 12 15/30 Write block 492 len 40960 data: 11 13 12 17 18 16 14 16 14 11 16/30 Write block 156 len 28672 data: 18 17 12 7 9 11 9 17/30 Read block 497 len 28672 18/30 Read block 149 len 20480 19/30 Read block 215 len 12288 20/30 Read block 501 len 12288 21/30 Read block 485 len 12288 22/30 Write block 366 len 12288 data: 14 14 12 23/30 Read block 197 len 40960 24/30 Write block 323 len 32768 data: 15 15 15 17 17 16 12 11 25/30 Read block 351 len 4096 26/30 Write block 159 len 28672 data: 8 10 12 10 10 14 13 27/30 Read block 381 len 20480 28/30 Write block 187 len 12288 data: 12 13 11 29/30 Read block 440 len 36864 30/30 Read block 167 len 36864 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:41.437679985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 150 150 150 247 247 247 T T T 3 060-079 150 150 150 247 247 247 T T T 4 080-099 154 154 154 254 254 254 F F F 5 100-119 154 154 154 254 254 254 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 148 148 148 244 244 244 T T T 8 160-179 155 155 155 257 257 257 T T T 9 180-199 155 155 155 257 257 257 T T T 10 200-219 147 147 147 242 242 242 T T T 11 220-239 154 154 154 254 254 254 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 152 152 152 252 252 252 T T T 17 340-359 153 153 153 253 253 253 F F F 18 360-379 151 151 151 250 250 250 T T T 19 380-399 154 154 154 254 254 254 F F F 20 400-419 152 152 152 252 252 252 T T T 21 420-439 154 154 154 256 256 256 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 154 154 154 255 255 255 T T T 24 480-499 152 152 152 251 251 251 T T T 25 500-519 150 150 150 247 247 247 T T T 26 520-539 155 155 155 257 257 257 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 154 154 154 256 256 256 T T T 29 580-599 154 154 154 256 256 256 F F F Max gen: 155, Max flush: 257 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 156 156 156 258 258 258 F F F 3 060-079 156 156 156 258 258 258 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 154 154 154 254 254 254 F F F 6 120-139 152 152 152 252 252 252 F F F 7 140-159 156 156 156 258 258 258 F F F 8 160-179 156 156 156 258 258 258 F F F 9 180-199 156 156 156 258 258 258 F F F 10 200-219 156 156 156 258 258 258 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 F F F 14 280-299 152 152 152 251 251 251 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 154 154 154 254 254 254 F F F 20 400-419 156 156 156 258 258 258 F F F 21 420-439 154 154 154 256 256 256 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 156 156 156 258 258 258 F F F 24 480-499 156 156 156 258 258 258 F F F 25 500-519 156 156 156 258 258 258 F F F 26 520-539 155 155 155 257 257 257 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 156 156 156 258 258 258 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 156, Max flush: 258 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.71679533Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.761116212Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.761809266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.761847601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.761862969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.761879027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} {"msg":"Crucible 6f1e0954-814f-43fd-9abf-1279fcd7bfba has session id: 72365d54-2d30-4858-a4e7-b0bb926bf644","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762384494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Upstairs opts: Upstairs UUID: 6f1e0954-814f-43fd-9abf-1279fcd7bfba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762423639Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Crucible stats registered with UUID: 6f1e0954-814f-43fd-9abf-1279fcd7bfba","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762445537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:45.762459685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762828439Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762854456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} {"msg":"6f1e0954-814f-43fd-9abf-1279fcd7bfba active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762922867Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762941665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762965802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.762991409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763018485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763045822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763073969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763100935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763129322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763155159Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_connection connected from Ok(127.0.0.1:61597)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763184925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_connection connected from Ok(127.0.0.1:57912)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763209202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_connection connected from Ok(127.0.0.1:64823)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763474089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763884258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.763913674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764076934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764153095Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764307765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764339631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764597939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764628376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.764901811Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765163059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765190946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765206414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765246559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765281014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765300322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765447334Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.76547566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[0]R flush_numbers[0..12]: [245, 254, 258, 258, 254, 254, 252, 258, 258, 258, 258, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765492128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[0]R generation[0..12]: [148, 154, 156, 156, 154, 154, 152, 156, 156, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765507536Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765522944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[1]R flush_numbers[0..12]: [245, 254, 258, 258, 254, 254, 252, 258, 258, 258, 258, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765537942Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[1]R generation[0..12]: [148, 154, 156, 156, 154, 154, 152, 156, 156, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.76555271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765567739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[2]R flush_numbers[0..12]: [245, 254, 258, 258, 254, 254, 252, 258, 258, 258, 258, 254]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765582697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[2]R generation[0..12]: [148, 154, 156, 156, 154, 154, 152, 156, 156, 156, 156, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765597255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765611293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765624612Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.76563819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765655888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765670446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid"Wait for a query_work_queue command to finish before sending IO :1814,"":"downstairs","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765690923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765703552Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"0","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.76571737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"1","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765730788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"client":"2","":"downstairs","session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765744307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765756705Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"6f1e0954-814f-43fd-9abf-1279fcd7bfba is now active with session: 72365d54-2d30-4858-a4e7-b0bb926bf644","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765769674Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.765781842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814,"session_id":"72365d54-2d30-4858-a4e7-b0bb926bf644"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:45.76579414Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 212 len 8192 data: 15 15 02/30 Flush 03/30 Write block 502 len 16384 data: 13 15 15 14 04/30 Write block 431 len 8192 data: 7 9 05/30 Read block 568 len 12288 06/30 Flush 07/30 Write block 236 len 4096 data: 13 08/30 Write block 108 len 36864 data: 10 10 11 14 17 17 19 19 17 09/30 Write block 51 len 28672 data: 12 10 12 10 11 12 13 10/30 Read block 545 len 24576 11/30 Flush 12/30 Write block 127 len 4096 data: 14 13/30 Flush 14/30 Write block 535 len 16384 data: 15 15 12 12 15/30 Write block 262 len 40960 data: 12 12 14 14 12 12 10 9 6 5 16/30 Write block 495 len 40960 data: 18 19 17 15 17 15 12 14 16 16 17/30 Read block 60 len 28672 18/30 Write block 57 len 28672 data: 14 15 19 16 19 18 13 19/30 Read block 114 len 32768 20/30 Read block 454 len 24576 21/30 Read block 400 len 40960 22/30 Write block 275 len 32768 data: 11 10 9 10 10 8 9 13 23/30 Flush 24/30 Write block 106 len 20480 data: 12 9 11 11 12 25/30 Flush 26/30 Write block 290 len 32768 data: 12 14 14 12 12 15 17 14 27/30 Write block 399 len 20480 data: 20 18 17 16 15 28/30 Read block 278 len 8192 29/30 Write block 66 len 16384 data: 11 11 14 10 30/30 Write block 486 len 12288 data: 11 12 12 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:45.784229229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 T T T 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 158 158 158 262 262 262 F F F 7 140-159 156 156 156 258 258 258 F F F 8 160-179 156 156 156 258 258 258 F F F 9 180-199 156 156 156 258 258 258 F F F 10 200-219 158 158 158 259 259 259 F F F 11 220-239 158 158 158 261 261 261 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 158 158 158 263 263 263 F F F 14 280-299 158 158 158 263 263 263 T T T 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 154 154 154 254 254 254 T T T 20 400-419 156 156 156 258 258 258 T T T 21 420-439 158 158 158 260 260 260 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 156 156 156 258 258 258 F F F 24 480-499 158 158 158 263 263 263 T T T 25 500-519 158 158 158 263 263 263 F F F 26 520-539 158 158 158 263 263 263 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 156 156 156 258 258 258 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 158, Max flush: 264 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 154 154 154 254 254 254 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 158 158 158 262 262 262 F F F 7 140-159 156 156 156 258 258 258 F F F 8 160-179 156 156 156 258 258 258 F F F 9 180-199 156 156 156 258 258 258 F F F 10 200-219 158 158 158 259 259 259 F F F 11 220-239 158 158 158 261 261 261 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 158 158 158 263 263 263 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 159 159 159 265 265 265 F F F 20 400-419 159 159 159 265 265 265 F F F 21 420-439 158 158 158 260 260 260 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 156 156 156 258 258 258 F F F 24 480-499 159 159 159 265 265 265 F F F 25 500-519 158 158 158 263 263 263 F F F 26 520-539 158 158 158 263 263 263 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 156 156 156 258 258 258 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 159, Max flush: 265 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.033955206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.078174421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.078847827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.078884063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.078898261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.078913039Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} {"msg":"Crucible 62eced1f-eca3-4cd6-82a1-77cb6a527ce8 has session id: 8a326508-fbb5-4d42-be86-55f47eb40c21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.079420736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Upstairs opts: Upstairs UUID: 62eced1f-eca3-4cd6-82a1-77cb6a527ce8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.07946432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Crucible stats registered with UUID: 62eced1f-eca3-4cd6-82a1-77cb6a527ce8","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.079492697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:50.079519694Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.079990775Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080019731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} {"msg":"62eced1f-eca3-4cd6-82a1-77cb6a527ce8 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080073215Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080101631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080137747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080169103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080200629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080228485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080258572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080285558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080343001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080384676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:42120)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080468866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:33726)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080499802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_connection connected from Ok(127.0.0.1:56646)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.080525029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081064242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081103007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081133223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.08116123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.08155923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081601565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081712651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.081785392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082144747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082326825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.08236435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082389327Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082411824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082435451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082458888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082704448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082732314Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[0]R flush_numbers[0..12]: [245, 254, 263, 265, 254, 264, 262, 258, 258, 258, 259, 261]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082753012Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[0]R generation[0..12]: [148, 154, 158, 159, 154, 158, 158, 156, 156, 156, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.08276854Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082784408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[1]R flush_numbers[0..12]: [245, 254, 263, 265, 254, 264, 262, 258, 258, 258, 259, 261]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082799806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[1]R generation[0..12]: [148, 154, 158, 159, 154, 158, 158, 156, 156, 156, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082814944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082829532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[2]R flush_numbers[0..12]: [245, 254, 263, 265, 254, 264, 262, 258, 258, 258, 259, 261]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082843661Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[2]R generation[0..12]: [148, 154, 158, 159, 154, 158, 158, 156, 156, 156, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082857669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082872627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082886055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082900604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082914212Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.08292755Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"":"downstairs","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082947238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082962046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"0","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082976974Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"1","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.082990812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"client":"2","":"downstairs","session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.083004611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.083017399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"62eced1f-eca3-4cd6-82a1-77cb6a527ce8 is now active with session: 8a326508-fbb5-4d42-be86-55f47eb40c21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.083031237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.083043826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822,"session_id":"8a326508-fbb5-4d42-be86-55f47eb40c21"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:50.083056424Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 8192 data: 10 9 02/30 Read block 301 len 28672 03/30 Write block 422 len 28672 data: 13 16 18 18 18 15 17 04/30 Read block 541 len 32768 05/30 Read block 48 len 20480 06/30 Write block 30 len 40960 data: 14 14 14 14 15 14 15 12 13 15 07/30 Write block 285 len 8192 data: 14 10 08/30 Read block 426 len 8192 09/30 Flush 10/30 Write block 396 len 4096 data: 17 11/30 Write block 470 len 8192 data: 11 10 12/30 Write block 410 len 8192 data: 10 10 13/30 Write block 155 len 28672 data: 18 19 18 13 9 11 13 14/30 Write block 500 len 28672 data: 16 13 15 17 17 15 16 15/30 Read block 305 len 40960 16/30 Read block 208 len 24576 17/30 Write block 268 len 8192 data: 11 10 18/30 Read block 357 len 40960 19/30 Write block 498 len 32768 data: 16 18 17 14 16 18 18 16 20/30 Read block 433 len 16384 21/30 Write block 466 len 16384 data: 15 13 12 11 22/30 Read block 299 len 20480 23/30 Write block 414 len 40960 data: 12 15 13 16 12 12 11 14 14 17 24/30 Read block 291 len 4096 25/30 Read block 343 len 36864 26/30 Write block 233 len 16384 data: 13 13 13 14 27/30 Write block 432 len 32768 data: 10 10 12 15 13 11 14 17 28/30 Read block 207 len 16384 29/30 Read block 483 len 20480 30/30 Write block 265 len 24576 data: 15 13 13 12 11 7 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:50.102297353Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 160 160 160 266 266 266 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 158 158 158 262 262 262 F F F 7 140-159 156 156 156 258 258 258 T T T 8 160-179 156 156 156 258 258 258 T T T 9 180-199 160 160 160 266 266 266 F F F 10 200-219 158 158 158 259 259 259 F F F 11 220-239 158 158 158 261 261 261 T T T 12 240-259 154 154 154 256 256 256 F F F 13 260-279 158 158 158 263 263 263 T T T 14 280-299 160 160 160 266 266 266 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 159 159 159 265 265 265 T T T 20 400-419 159 159 159 265 265 265 T T T 21 420-439 160 160 160 266 266 266 T T T 22 440-459 147 147 147 242 242 242 F F F 23 460-479 156 156 156 258 258 258 T T T 24 480-499 159 159 159 265 265 265 T T T 25 500-519 158 158 158 263 263 263 T T T 26 520-539 158 158 158 263 263 263 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 156 156 156 258 258 258 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 160, Max flush: 266 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 F F F 1 020-039 160 160 160 266 266 266 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 158 158 158 262 262 262 F F F 7 140-159 161 161 161 267 267 267 F F F 8 160-179 161 161 161 267 267 267 F F F 9 180-199 160 160 160 266 266 266 F F F 10 200-219 158 158 158 259 259 259 F F F 11 220-239 161 161 161 267 267 267 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 161 161 161 267 267 267 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 161 161 161 267 267 267 F F F 20 400-419 161 161 161 267 267 267 F F F 21 420-439 161 161 161 267 267 267 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 161 161 161 267 267 267 F F F 24 480-499 161 161 161 267 267 267 F F F 25 500-519 161 161 161 267 267 267 F F F 26 520-539 158 158 158 263 263 263 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 156 156 156 258 258 258 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 161, Max flush: 267 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.369060812Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:54.411124535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.411493779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.442337386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.442942931Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.442980196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.442998394Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.443024321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} {"msg":"Crucible 47c0dfa4-2311-4911-ba45-76d9d8cd8589 has session id: ede26b14-0c79-47a0-8e03-f903c81f8d0d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44343609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Upstairs opts: Upstairs UUID: 47c0dfa4-2311-4911-ba45-76d9d8cd8589, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.443480644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Crucible stats registered with UUID: 47c0dfa4-2311-4911-ba45-76d9d8cd8589","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.443509911Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:54.443533208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.443942477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.443968723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} {"msg":"47c0dfa4-2311-4911-ba45-76d9d8cd8589 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44399792Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444011748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444045964Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444062252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44407647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444089888Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444103617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444116955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444179357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444201005Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35980)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444329949Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38124)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444370284Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48838)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444404269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.444994796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445022923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44503875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445053299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445073106Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445088214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445629097Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445658993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.445805435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446210185Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446234172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44624966Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44633145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446351207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446368825Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446382574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446396032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[0]R flush_numbers[0..12]: [245, 266, 263, 265, 254, 264, 262, 267, 267, 266, 259, 267]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44640986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[0]R generation[0..12]: [148, 160, 158, 159, 154, 158, 158, 161, 161, 160, 158, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446424018Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446438267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[1]R flush_numbers[0..12]: [245, 266, 263, 265, 254, 264, 262, 267, 267, 266, 259, 267]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446452205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[1]R generation[0..12]: [148, 160, 158, 159, 154, 158, 158, 161, 161, 160, 158, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446465973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446480242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[2]R flush_numbers[0..12]: [245, 266, 263, 265, 254, 264, 262, 267, 267, 266, 259, 267]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44649421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[2]R generation[0..12]: [148, 160, 158, 159, 154, 158, 158, 161, 161, 160, 158, 161]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446507958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446521976Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446535184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446548463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446563731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446577019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446596737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446609275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"0","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446622824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"1","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446636242Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"client":"2","":"downstairs","session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.44664954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446661869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"47c0dfa4-2311-4911-ba45-76d9d8cd8589 is now active with session: ede26b14-0c79-47a0-8e03-f903c81f8d0d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446674787Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446687036Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830,"session_id":"ede26b14-0c79-47a0-8e03-f903c81f8d0d"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:54.446699214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 28672 data: 22 21 20 20 18 16 15 02/30 Write block 234 len 24576 data: 14 14 15 9 10 10 03/30 Read block 315 len 8192 04/30 Read block 349 len 28672 05/30 Write block 194 len 28672 data: 15 13 12 11 15 14 15 06/30 Read block 67 len 24576 07/30 Read block 107 len 28672 08/30 Read block 38 len 4096 09/30 Read block 349 len 40960 10/30 Read block 169 len 16384 11/30 Read block 24 len 8192 12/30 Write block 429 len 24576 data: 14 9 8 11 11 13 13/30 Write block 534 len 24576 data: 13 16 16 13 13 12 14/30 Write block 135 len 4096 data: 11 15/30 Write block 558 len 8192 data: 13 16 16/30 Read block 78 len 4096 17/30 Write block 560 len 12288 data: 14 13 14 18/30 Read block 485 len 28672 19/30 Read block 346 len 20480 20/30 Read block 10 len 4096 21/30 Read block 552 len 20480 22/30 Read block 22 len 32768 23/30 Write block 40 len 12288 data: 14 12 13 24/30 Write block 141 len 28672 data: 10 9 9 10 14 12 10 25/30 Write block 156 len 4096 data: 20 26/30 Write block 10 len 8192 data: 16 16 27/30 Read block 529 len 40960 28/30 Write block 478 len 40960 data: 17 17 17 17 17 17 13 11 12 13 29/30 Write block 569 len 8192 data: 7 9 30/30 Read block 214 len 4096 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:54.464473765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 245 245 245 T T T 1 020-039 160 160 160 266 266 266 T T T 2 040-059 158 158 158 263 263 263 T T T 3 060-079 159 159 159 265 265 265 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 158 158 158 262 262 262 T T T 7 140-159 161 161 161 267 267 267 T T T 8 160-179 161 161 161 267 267 267 F F F 9 180-199 160 160 160 266 266 266 T T T 10 200-219 158 158 158 259 259 259 T T T 11 220-239 161 161 161 267 267 267 T T T 12 240-259 154 154 154 256 256 256 F F F 13 260-279 161 161 161 267 267 267 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 161 161 161 267 267 267 F F F 20 400-419 161 161 161 267 267 267 F F F 21 420-439 161 161 161 267 267 267 T T T 22 440-459 147 147 147 242 242 242 F F F 23 460-479 161 161 161 267 267 267 T T T 24 480-499 161 161 161 267 267 267 T T T 25 500-519 161 161 161 267 267 267 F F F 26 520-539 158 158 158 263 263 263 T T T 27 540-559 152 152 152 251 251 251 T T T 28 560-579 156 156 156 258 258 258 T T T 29 580-599 154 154 154 256 256 256 F F F Max gen: 161, Max flush: 267 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 162 162 162 268 268 268 F F F 2 040-059 162 162 162 268 268 268 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 158 158 158 264 264 264 F F F 6 120-139 162 162 162 268 268 268 F F F 7 140-159 162 162 162 268 268 268 F F F 8 160-179 161 161 161 267 267 267 F F F 9 180-199 162 162 162 268 268 268 F F F 10 200-219 162 162 162 268 268 268 F F F 11 220-239 162 162 162 268 268 268 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 161 161 161 267 267 267 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 154 154 154 256 256 256 F F F 16 320-339 156 156 156 258 258 258 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 156 156 156 258 258 258 F F F 19 380-399 161 161 161 267 267 267 F F F 20 400-419 161 161 161 267 267 267 F F F 21 420-439 162 162 162 268 268 268 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 162 162 162 268 268 268 F F F 24 480-499 162 162 162 268 268 268 F F F 25 500-519 161 161 161 267 267 267 F F F 26 520-539 162 162 162 268 268 268 F F F 27 540-559 162 162 162 268 268 268 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 162, Max flush: 268 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.785937817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.829728245Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830436567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830472893Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830487631Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830502829Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} {"msg":"Crucible 76946339-6d80-4a7a-89b4-f3198f3c9e83 has session id: c48fe81f-9e7e-4f14-b62e-52006626da17","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830891521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Upstairs opts: Upstairs UUID: 76946339-6d80-4a7a-89b4-f3198f3c9e83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830934725Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Crucible stats registered with UUID: 76946339-6d80-4a7a-89b4-f3198f3c9e83","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.830961152Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:58.830985529Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831467059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831492696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} {"msg":"76946339-6d80-4a7a-89b4-f3198f3c9e83 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831555248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831573866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831602762Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831631609Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831660465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831690931Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831718368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831748044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831778121Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831805867Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_connection connected from Ok(127.0.0.1:45560)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831836753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_connection connected from Ok(127.0.0.1:53931)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83186701Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_connection connected from Ok(127.0.0.1:59969)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.831895166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.832532557Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.832562303Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.832578681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.832596459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833268735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83330935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833753585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833780172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833938672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833968408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.833994015Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834014292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83403255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834046739Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834362729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834875996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834895403Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[0]R flush_numbers[0..12]: [268, 268, 268, 265, 254, 264, 268, 268, 267, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834915511Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[0]R generation[0..12]: [162, 162, 162, 159, 154, 158, 162, 162, 161, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834929619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834943707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[1]R flush_numbers[0..12]: [268, 268, 268, 265, 254, 264, 268, 268, 267, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834957895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[1]R generation[0..12]: [162, 162, 162, 159, 154, 158, 162, 162, 161, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834971844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.834985822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[2]R flush_numbers[0..12]: [268, 268, 268, 265, 254, 264, 268, 268, 267, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83499964Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[2]R generation[0..12]: [162, 162, 162, 159, 154, 158, 162, 162, 161, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835013718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835027517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Max found gen is 163","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835043655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Generation requested: 164 >= found:163","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835057173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835070681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83508406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"":"downstairs","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835097468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835109807Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835176688Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"1","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835193876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"client":"2","":"downstairs","session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835207444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835219883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"76946339-6d80-4a7a-89b4-f3198f3c9e83 is now active with session: c48fe81f-9e7e-4f14-b62e-52006626da17","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835232831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.83524506Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838,"session_id":"c48fe81f-9e7e-4f14-b62e-52006626da17"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:47:58.835257638Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 412 len 20480 data: 9 12 13 16 14 02/30 Write block 378 len 24576 data: 11 10 10 10 8 9 03/30 Write block 130 len 40960 data: 15 11 11 12 9 12 10 10 11 12 04/30 Read block 219 len 28672 05/30 Write block 43 len 8192 data: 15 13 06/30 Flush 07/30 Write block 268 len 4096 data: 13 08/30 Write block 306 len 40960 data: 8 8 8 9 10 10 10 15 12 11 09/30 Read block 20 len 32768 10/30 Read block 80 len 4096 11/30 Write block 57 len 20480 data: 15 16 20 17 20 12/30 Write block 188 len 12288 data: 14 12 9 13/30 Write block 178 len 32768 data: 19 18 18 18 15 15 14 13 14/30 Write block 494 len 4096 data: 13 15/30 Flush 16/30 Write block 31 len 24576 data: 15 15 15 16 15 16 17/30 Write block 59 len 12288 data: 21 18 21 18/30 Flush 19/30 Write block 375 len 8192 data: 7 8 20/30 Write block 113 len 28672 data: 18 20 20 18 15 13 14 21/30 Flush 22/30 Write block 265 len 12288 data: 16 14 14 23/30 Read block 230 len 20480 24/30 Write block 130 len 24576 data: 16 12 12 13 10 13 25/30 Write block 393 len 36864 data: 11 13 17 18 16 18 21 19 18 26/30 Write block 51 len 4096 data: 13 27/30 Write block 325 len 4096 data: 16 28/30 Read block 265 len 40960 29/30 Read block 236 len 28672 30/30 Write block 509 len 32768 data: 15 16 14 15 15 12 11 9 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:47:58.852971897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 164 164 164 271 271 271 T T T 3 060-079 164 164 164 271 271 271 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 164 164 164 272 272 272 F F F 6 120-139 164 164 164 269 269 269 T T T 7 140-159 162 162 162 268 268 268 F F F 8 160-179 164 164 164 270 270 270 F F F 9 180-199 164 164 164 270 270 270 F F F 10 200-219 162 162 162 268 268 268 F F F 11 220-239 162 162 162 268 268 268 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 164 164 164 270 270 270 T T T 14 280-299 160 160 160 266 266 266 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 156 156 156 258 258 258 T T T 17 340-359 153 153 153 253 253 253 F F F 18 360-379 164 164 164 272 272 272 F F F 19 380-399 164 164 164 269 269 269 T T T 20 400-419 164 164 164 269 269 269 T T T 21 420-439 162 162 162 268 268 268 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 162 162 162 268 268 268 F F F 24 480-499 164 164 164 270 270 270 F F F 25 500-519 161 161 161 267 267 267 T T T 26 520-539 162 162 162 268 268 268 F F F 27 540-559 162 162 162 268 268 268 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 164, Max flush: 272 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 165 165 165 273 273 273 F F F 3 060-079 164 164 164 271 271 271 F F F 4 080-099 154 154 154 254 254 254 F F F 5 100-119 164 164 164 272 272 272 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 162 162 162 268 268 268 F F F 8 160-179 164 164 164 270 270 270 F F F 9 180-199 164 164 164 270 270 270 F F F 10 200-219 162 162 162 268 268 268 F F F 11 220-239 162 162 162 268 268 268 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 165 165 165 273 273 273 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 164 164 164 272 272 272 F F F 19 380-399 165 165 165 273 273 273 F F F 20 400-419 165 165 165 273 273 273 F F F 21 420-439 162 162 162 268 268 268 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 162 162 162 268 268 268 F F F 24 480-499 164 164 164 270 270 270 F F F 25 500-519 165 165 165 273 273 273 F F F 26 520-539 162 162 162 268 268 268 F F F 27 540-559 162 162 162 268 268 268 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 154 154 154 256 256 256 F F F Max gen: 165, Max flush: 273 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.111342449Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.154276844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.154881368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.154917234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.154931982Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15494659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} {"msg":"Crucible bf4304b0-9ece-4b7e-bb15-3c3f7689bd11 has session id: 483f76a0-2d5c-46b1-ad24-8ebd4cd95156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.155372367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Upstairs opts: Upstairs UUID: bf4304b0-9ece-4b7e-bb15-3c3f7689bd11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.155416932Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Crucible stats registered with UUID: bf4304b0-9ece-4b7e-bb15-3c3f7689bd11","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.155449998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:03.155472875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.155933698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.155968423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} {"msg":"bf4304b0-9ece-4b7e-bb15-3c3f7689bd11 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15599438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156016027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156047763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15607163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156095167Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156118604Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156142732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156166449Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156189706Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156212413Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_connection connected from Ok(127.0.0.1:42934)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156249758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_connection connected from Ok(127.0.0.1:52951)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156278755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_connection connected from Ok(127.0.0.1:34848)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156303052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156926944Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.156954871Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.157063007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.157124499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.157418183Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.157534539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.157980273Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.1580091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158024858Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158110727Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158140293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158155842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158460054Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.158479291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15897049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159441162Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159461139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[0]R flush_numbers[0..12]: [268, 271, 273, 271, 254, 272, 273, 268, 270, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159475627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[0]R generation[0..12]: [162, 164, 165, 164, 154, 164, 165, 162, 164, 164, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159489686Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159504874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[1]R flush_numbers[0..12]: [268, 271, 273, 271, 254, 272, 273, 268, 270, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159518912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[1]R generation[0..12]: [162, 164, 165, 164, 154, 164, 165, 162, 164, 164, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15953277Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159546869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[2]R flush_numbers[0..12]: [268, 271, 273, 271, 254, 272, 273, 268, 270, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159560627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[2]R generation[0..12]: [162, 164, 165, 164, 154, 164, 165, 162, 164, 164, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159574425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159588323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159601432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.15961784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159631158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159644097Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"":"downstairs","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159657765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159670033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"0","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159690351Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"1","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159703869Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"client":"2","":"downstairs","session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159717517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159730046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"bf4304b0-9ece-4b7e-bb15-3c3f7689bd11 is now active with session: 483f76a0-2d5c-46b1-ad24-8ebd4cd95156","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159743064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159755253Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846,"session_id":"483f76a0-2d5c-46b1-ad24-8ebd4cd95156"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:03.159767501Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 4096 data: 16 02/30 Read block 9 len 32768 03/30 Write block 304 len 28672 data: 10 9 9 9 9 10 11 04/30 Read block 570 len 12288 05/30 Flush 06/30 Write block 434 len 28672 data: 14 16 14 12 15 18 15 07/30 Write block 145 len 4096 data: 15 08/30 Write block 584 len 16384 data: 8 6 7 10 09/30 Write block 465 len 16384 data: 15 16 14 13 10/30 Read block 122 len 28672 11/30 Read block 393 len 24576 12/30 Write block 147 len 28672 data: 11 12 14 13 13 18 19 13/30 Read block 179 len 20480 14/30 Write block 306 len 8192 data: 10 10 15/30 Flush 16/30 Write block 591 len 12288 data: 12 11 11 17/30 Flush 18/30 Write block 168 len 20480 data: 11 10 8 8 9 19/30 Write block 223 len 4096 data: 12 20/30 Read block 277 len 12288 21/30 Read block 507 len 36864 22/30 Write block 539 len 12288 data: 13 11 11 23/30 Write block 151 len 16384 data: 14 19 20 19 24/30 Read block 406 len 40960 25/30 Write block 373 len 32768 data: 10 8 8 9 11 12 11 11 26/30 Write block 190 len 28672 data: 10 11 10 13 16 14 13 27/30 Write block 87 len 28672 data: 15 16 17 13 10 11 9 28/30 Read block 33 len 36864 29/30 Read block 129 len 32768 30/30 Write block 216 len 40960 data: 14 13 11 7 11 11 12 13 13 15 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:03.178511002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 165 165 165 273 273 273 F F F 3 060-079 164 164 164 271 271 271 F F F 4 080-099 154 154 154 254 254 254 T T T 5 100-119 164 164 164 272 272 272 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 166 166 166 275 275 275 T T T 8 160-179 164 164 164 270 270 270 T T T 9 180-199 164 164 164 270 270 270 T T T 10 200-219 162 162 162 268 268 268 T T T 11 220-239 162 162 162 268 268 268 T T T 12 240-259 154 154 154 256 256 256 F F F 13 260-279 165 165 165 273 273 273 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 164 164 164 272 272 272 T T T 19 380-399 165 165 165 273 273 273 T T T 20 400-419 165 165 165 273 273 273 F F F 21 420-439 166 166 166 275 275 275 F F F 22 440-459 166 166 166 275 275 275 F F F 23 460-479 166 166 166 275 275 275 F F F 24 480-499 164 164 164 270 270 270 F F F 25 500-519 165 165 165 273 273 273 F F F 26 520-539 162 162 162 268 268 268 T T T 27 540-559 162 162 162 268 268 268 T T T 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 166, Max flush: 276 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 165 165 165 273 273 273 F F F 3 060-079 164 164 164 271 271 271 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 164 164 164 272 272 272 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 167 167 167 277 277 277 F F F 8 160-179 167 167 167 277 277 277 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 167 167 167 277 277 277 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 165 165 165 273 273 273 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 167 167 167 277 277 277 F F F 19 380-399 167 167 167 277 277 277 F F F 20 400-419 165 165 165 273 273 273 F F F 21 420-439 166 166 166 275 275 275 F F F 22 440-459 166 166 166 275 275 275 F F F 23 460-479 166 166 166 275 275 275 F F F 24 480-499 164 164 164 270 270 270 F F F 25 500-519 165 165 165 273 273 273 F F F 26 520-539 167 167 167 277 277 277 F F F 27 540-559 167 167 167 277 277 277 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 167, Max flush: 277 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.44874666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:07.490864226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.491249598Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.521958442Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.522548379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.522583784Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.522598882Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.522613671Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} {"msg":"Crucible b5d5e6a4-b37b-4b40-a98a-0e7cf58bb426 has session id: a2828649-456a-40f6-a17b-7e032aa53172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523057025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Upstairs opts: Upstairs UUID: b5d5e6a4-b37b-4b40-a98a-0e7cf58bb426, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52310254Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Crucible stats registered with UUID: b5d5e6a4-b37b-4b40-a98a-0e7cf58bb426","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523131096Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:07.523149904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523627715Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523648802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} {"msg":"b5d5e6a4-b37b-4b40-a98a-0e7cf58bb426 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523721563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523738981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523761498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523775836Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523789585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523803083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52382511Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523849527Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523875104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.523900021Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_connection connected from Ok(127.0.0.1:51950)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524086108Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_connection connected from Ok(127.0.0.1:33691)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524123353Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_connection connected from Ok(127.0.0.1:35776)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52414794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524598384Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524639629Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524671145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524699681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524778692Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.524801709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525295398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525327163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525343672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525881565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52592069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525939477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525952696Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525970934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.525985042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.5259975Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526011238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[0]R flush_numbers[0..12]: [268, 271, 273, 271, 277, 272, 273, 277, 277, 277, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526025187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[0]R generation[0..12]: [162, 164, 165, 164, 167, 164, 165, 167, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526039255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526053543Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[1]R flush_numbers[0..12]: [268, 271, 273, 271, 277, 272, 273, 277, 277, 277, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526067391Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[1]R generation[0..12]: [162, 164, 165, 164, 167, 164, 165, 167, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52608133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526095468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[2]R flush_numbers[0..12]: [268, 271, 273, 271, 277, 272, 273, 277, 277, 277, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526109386Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[2]R generation[0..12]: [162, 164, 165, 164, 167, 164, 165, 167, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526123075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526137173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526166049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526183767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526198565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526211833Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"":"downstairs","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526225142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV"Wait for a query_work_queue command to finish before sending IO ,"pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52624382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"0","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526257128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"1","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526270546Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"client":"2","":"downstairs","session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526284255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526296763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"b5d5e6a4-b37b-4b40-a98a-0e7cf58bb426 is now active with session: a2828649-456a-40f6-a17b-7e032aa53172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526309681Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.52632195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854,"session_id":"a2828649-456a-40f6-a17b-7e032aa53172"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:07.526334358Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 540 len 12288 data: 12 12 11 02/30 Read block 257 len 4096 03/30 Read block 580 len 20480 04/30 Write block 460 len 24576 data: 15 12 13 11 12 16 05/30 Write block 424 len 8192 data: 19 19 06/30 Read block 557 len 16384 07/30 Read block 581 len 28672 08/30 Write block 493 len 36864 data: 14 14 19 20 18 17 19 18 15 09/30 Write block 485 len 40960 data: 12 13 14 13 13 13 9 12 15 15 10/30 Read block 556 len 36864 11/30 Read block 563 len 32768 12/30 Write block 261 len 20480 data: 10 13 13 15 17 13/30 Flush 14/30 Write block 224 len 8192 data: 14 16 15/30 Read block 278 len 40960 16/30 Read block 325 len 12288 17/30 Read block 324 len 36864 18/30 Read block 374 len 20480 19/30 Read block 16 len 32768 20/30 Write block 419 len 24576 data: 13 12 15 15 18 20 21/30 Write block 470 len 40960 data: 12 11 14 13 12 15 15 16 18 18 22/30 Read block 374 len 12288 23/30 Write block 254 len 20480 data: 10 8 12 12 11 24/30 Write block 148 len 40960 data: 13 15 14 15 20 21 20 19 21 19 25/30 Write block 278 len 8192 data: 11 11 26/30 Write block 40 len 12288 data: 15 13 14 27/30 Write block 497 len 16384 data: 19 18 20 19 28/30 Write block 111 len 24576 data: 15 18 19 21 21 19 29/30 Write block 492 len 8192 data: 13 16 30/30 Read block 195 len 4096 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:07.545436104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 165 165 165 273 273 273 T T T 3 060-079 164 164 164 271 271 271 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 164 164 164 272 272 272 T T T 6 120-139 165 165 165 273 273 273 F F F 7 140-159 167 167 167 277 277 277 T T T 8 160-179 167 167 167 277 277 277 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 167 167 167 277 277 277 T T T 12 240-259 154 154 154 256 256 256 T T T 13 260-279 168 168 168 278 278 278 T T T 14 280-299 160 160 160 266 266 266 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 167 167 167 277 277 277 F F F 19 380-399 167 167 167 277 277 277 F F F 20 400-419 165 165 165 273 273 273 T T T 21 420-439 168 168 168 278 278 278 T T T 22 440-459 166 166 166 275 275 275 F F F 23 460-479 168 168 168 278 278 278 T T T 24 480-499 168 168 168 278 278 278 T T T 25 500-519 168 168 168 278 278 278 T T T 26 520-539 167 167 167 277 277 277 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 168, Max flush: 278 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 268 268 268 F F F 1 020-039 164 164 164 271 271 271 F F F 2 040-059 169 169 169 279 279 279 F F F 3 060-079 164 164 164 271 271 271 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 169 169 169 279 279 279 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 169 169 169 279 279 279 F F F 8 160-179 167 167 167 277 277 277 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 F F F 13 260-279 169 169 169 279 279 279 F F F 14 280-299 160 160 160 266 266 266 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 167 167 167 277 277 277 F F F 19 380-399 167 167 167 277 277 277 F F F 20 400-419 169 169 169 279 279 279 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 166 166 166 275 275 275 F F F 23 460-479 169 169 169 279 279 279 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 169 169 169 279 279 279 F F F 26 520-539 167 167 167 277 277 277 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 169, Max flush: 279 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.851847896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.896814718Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.897816104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.897857689Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.897872647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.897887765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} {"msg":"Crucible f91160d6-3818-4488-b1b9-e2efb6956b65 has session id: 29da24dd-5dc4-4f00-a4a0-bc9e0c938666","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.898359946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Upstairs opts: Upstairs UUID: f91160d6-3818-4488-b1b9-e2efb6956b65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.89841138Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Crucible stats registered with UUID: f91160d6-3818-4488-b1b9-e2efb6956b65","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.898441226Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:11.898464803Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.89905538Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899086556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} {"msg":"f91160d6-3818-4488-b1b9-e2efb6956b65 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899155817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899175675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899202162Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.89921692Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899231328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899246526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899261114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899274692Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899288491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899302279Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_connection connected from Ok(127.0.0.1:39133)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899483507Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_connection connected from Ok(127.0.0.1:60263)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.899799127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:11.899975296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:11.900012081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:11.900032178Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.900163532Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.900348659Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.900379625Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.900450626Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.900496601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901000738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901155909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901567318Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901595434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901610312Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901635149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901649457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:11.901663006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.901461081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_connection connected from Ok(127.0.0.1:52563)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.901658126Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.902648153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.902691698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903171878Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903701193Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903726489Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[0]R flush_numbers[0..12]: [268, 271, 279, 271, 277, 279, 273, 279, 277, 277, 277, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903742537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[0]R generation[0..12]: [162, 164, 169, 164, 167, 169, 165, 169, 167, 167, 167, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903757535Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903772264Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[1]R flush_numbers[0..12]: [268, 271, 279, 271, 277, 279, 273, 279, 277, 277, 277, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903786872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[1]R generation[0..12]: [162, 164, 169, 164, 167, 169, 165, 169, 167, 167, 167, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.90380143Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903821187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[2]R flush_numbers[0..12]: [268, 271, 279, 271, 277, 279, 273, 279, 277, 277, 277, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903835876Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[2]R generation[0..12]: [162, 164, 169, 164, 167, 169, 165, 169, 167, 167, 167, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903850184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903864712Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.90387818Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903891809Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903905337Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903918945Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"":"downstairs","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903932864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903945252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"0","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.90395889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"1","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903972509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"client":"2","":"downstairs","session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903986207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.903998956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"f91160d6-3818-4488-b1b9-e2efb6956b65 is now active with session: 29da24dd-5dc4-4f00-a4a0-bc9e0c938666","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-12-11T23:48:21.904012324Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.904037061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862,"session_id":"29da24dd-5dc4-4f00-a4a0-bc9e0c938666"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:21.904050549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 8192 data: 22 23 02/30 Read block 528 len 32768 03/30 Read block 108 len 32768 04/30 Write block 105 len 4096 data: 12 05/30 Write block 164 len 24576 data: 15 14 12 14 12 11 06/30 Write block 466 len 40960 data: 17 15 14 12 13 12 15 14 13 16 07/30 Flush 08/30 Flush 09/30 Write block 46 len 40960 data: 12 11 12 10 12 14 11 13 11 12 10/30 Write block 9 len 40960 data: 17 17 17 17 17 22 20 18 17 19 11/30 Read block 59 len 8192 12/30 Read block 271 len 20480 13/30 Write block 73 len 8192 data: 14 13 14/30 Read block 40 len 32768 15/30 Write block 205 len 24576 data: 13 12 11 13 14 16 16/30 Write block 288 len 4096 data: 17 17/30 Write block 326 len 40960 data: 18 18 17 13 12 10 7 6 7 9 18/30 Write block 170 len 40960 data: 9 9 10 10 12 9 16 16 20 19 19/30 Read block 228 len 24576 20/30 Read block 241 len 16384 21/30 Write block 524 len 24576 data: 10 9 9 10 7 7 22/30 Write block 546 len 4096 data: 10 23/30 Flush 24/30 Write block 272 len 12288 data: 5 8 8 25/30 Read block 368 len 32768 26/30 Read block 250 len 12288 27/30 Read block 469 len 36864 28/30 Read block 481 len 24576 29/30 Read block 380 len 12288 30/30 Write block 90 len 36864 data: 14 11 12 10 9 9 10 10 8 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:21.924599755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 170 170 170 282 282 282 F F F 3 060-079 170 170 170 282 282 282 F F F 4 080-099 167 167 167 277 277 277 T T T 5 100-119 170 170 170 280 280 280 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 169 169 169 279 279 279 F F F 8 160-179 170 170 170 282 282 282 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 F F F 13 260-279 169 169 169 279 279 279 T T T 14 280-299 170 170 170 282 282 282 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 170 170 170 282 282 282 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 167 167 167 277 277 277 F F F 19 380-399 167 167 167 277 277 277 F F F 20 400-419 169 169 169 279 279 279 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 166 166 166 275 275 275 F F F 23 460-479 170 170 170 280 280 280 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 169 169 169 279 279 279 F F F 26 520-539 170 170 170 282 282 282 F F F 27 540-559 170 170 170 282 282 282 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 170, Max flush: 282 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 282 282 282 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 170 170 170 282 282 282 F F F 3 060-079 170 170 170 282 282 282 F F F 4 080-099 171 171 171 283 283 283 F F F 5 100-119 170 170 170 280 280 280 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 169 169 169 279 279 279 F F F 8 160-179 170 170 170 282 282 282 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 F F F 13 260-279 171 171 171 283 283 283 F F F 14 280-299 170 170 170 282 282 282 F F F 15 300-319 166 166 166 275 275 275 F F F 16 320-339 170 170 170 282 282 282 F F F 17 340-359 153 153 153 253 253 253 F F F 18 360-379 167 167 167 277 277 277 F F F 19 380-399 167 167 167 277 277 277 F F F 20 400-419 169 169 169 279 279 279 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 166 166 166 275 275 275 F F F 23 460-479 170 170 170 280 280 280 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 169 169 169 279 279 279 F F F 26 520-539 170 170 170 282 282 282 F F F 27 540-559 170 170 170 282 282 282 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 171, Max flush: 283 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.15076276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.195472164Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196305201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196339806Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196359884Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196375302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} {"msg":"Crucible 78130293-6548-46ef-a451-4101d7cf5e4f has session id: 1842c4a4-c88b-400e-a0f0-ac2edf2c5dca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196922914Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Upstairs opts: Upstairs UUID: 78130293-6548-46ef-a451-4101d7cf5e4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.19695369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Crucible stats registered with UUID: 78130293-6548-46ef-a451-4101d7cf5e4f","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.196969248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:26.196991195Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197639955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197668791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} {"msg":"78130293-6548-46ef-a451-4101d7cf5e4f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197718135Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197746861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197772758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197787987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197802385Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197816243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197831301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197849699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197865247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197879985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:61404)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.19792477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:61589)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197952676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_connection connected from Ok(127.0.0.1:45843)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.197980223Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198709432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198744457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198770734Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198800031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198875381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.198906827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.199334634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.199380589Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.199504733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.19993233Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.199968345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.199991043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200005761Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200019669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200033137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200099579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200118497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[0]R flush_numbers[0..12]: [282, 280, 282, 282, 283, 280, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200135445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[0]R generation[0..12]: [170, 170, 170, 170, 171, 170, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200150643Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200164971Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[1]R flush_numbers[0..12]: [282, 280, 282, 282, 283, 280, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200179189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[1]R generation[0..12]: [170, 170, 170, 170, 171, 170, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200193368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200207486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[2]R flush_numbers[0..12]: [282, 280, 282, 282, 283, 280, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200225213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[2]R generation[0..12]: [170, 170, 170, 170, 171, 170, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200239582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.20025376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200267468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200280927Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200294145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200307243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"":"downstairs","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:48:26.200320482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.20033723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"0","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200350738Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"1","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200364026Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"client":"2","":"downstairs","session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200377275Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200389923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"78130293-6548-46ef-a451-4101d7cf5e4f is now active with session: 1842c4a4-c88b-400e-a0f0-ac2edf2c5dca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200402922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.20041521Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870,"session_id":"1842c4a4-c88b-400e-a0f0-ac2edf2c5dca"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:26.200427698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 106 len 28672 data: 13 10 12 12 13 16 19 02/30 Read block 311 len 16384 03/30 Write block 12 len 28672 data: 18 18 23 21 19 18 20 04/30 Read block 327 len 28672 05/30 Write block 518 len 36864 data: 9 10 9 11 10 12 11 10 10 06/30 Write block 323 len 24576 data: 16 16 17 19 19 18 07/30 Read block 101 len 16384 08/30 Read block 225 len 16384 09/30 Read block 531 len 36864 10/30 Read block 32 len 20480 11/30 Read block 445 len 16384 12/30 Read block 18 len 40960 13/30 Write block 59 len 32768 data: 22 19 22 19 14 15 14 12 14/30 Write block 72 len 4096 data: 13 15/30 Read block 335 len 32768 16/30 Flush 17/30 Write block 83 len 36864 data: 12 15 15 17 16 17 18 15 12 18/30 Write block 418 len 4096 data: 13 19/30 Read block 515 len 8192 20/30 Read block 569 len 32768 21/30 Write block 311 len 12288 data: 11 11 16 22/30 Flush 23/30 Write block 494 len 4096 data: 16 24/30 Read block 461 len 16384 25/30 Flush 26/30 Write block 344 len 36864 data: 13 14 12 10 10 10 11 10 10 27/30 Write block 459 len 16384 data: 11 16 13 14 28/30 Read block 552 len 8192 29/30 Write block 379 len 32768 data: 12 12 11 9 10 6 6 7 30/30 Read block 237 len 28672 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 3 4 5 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:26.221161512Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 284 284 284 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 172 172 172 284 284 284 F F F 3 060-079 172 172 172 284 284 284 F F F 4 080-099 172 172 172 285 285 285 F F F 5 100-119 172 172 172 284 284 284 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 169 169 169 279 279 279 F F F 8 160-179 170 170 170 282 282 282 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 F F F 13 260-279 171 171 171 283 283 283 F F F 14 280-299 170 170 170 282 282 282 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 172 172 172 284 284 284 F F F 17 340-359 153 153 153 253 253 253 T T T 18 360-379 167 167 167 277 277 277 T T T 19 380-399 167 167 167 277 277 277 T T T 20 400-419 172 172 172 285 285 285 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 166 166 166 275 275 275 T T T 23 460-479 170 170 170 280 280 280 T T T 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 172 172 172 284 284 284 F F F 27 540-559 170 170 170 282 282 282 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 172, Max flush: 286 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 284 284 284 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 172 172 172 284 284 284 F F F 3 060-079 172 172 172 284 284 284 F F F 4 080-099 172 172 172 285 285 285 F F F 5 100-119 172 172 172 284 284 284 F F F 6 120-139 165 165 165 273 273 273 F F F 7 140-159 169 169 169 279 279 279 F F F 8 160-179 170 170 170 282 282 282 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 F F F 13 260-279 171 171 171 283 283 283 F F F 14 280-299 170 170 170 282 282 282 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 172 172 172 284 284 284 F F F 17 340-359 173 173 173 287 287 287 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 173 173 173 287 287 287 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 172 172 172 284 284 284 F F F 27 540-559 170 170 170 282 282 282 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 F F F Max gen: 173, Max flush: 287 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.461764872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.506657923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.507603666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.507642581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.507658129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.507674387Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} {"msg":"Crucible ac324125-2b8c-40a7-97af-809b83204a7d has session id: 14a9887e-0542-4bee-91f1-0032571da053","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508200852Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Upstairs opts: Upstairs UUID: ac324125-2b8c-40a7-97af-809b83204a7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508246786Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Crucible stats registered with UUID: ac324125-2b8c-40a7-97af-809b83204a7d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508275542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:30.508301709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508740114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508766871Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} {"msg":"ac324125-2b8c-40a7-97af-809b83204a7d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508829083Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508847981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508871808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508890996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508918432Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508947349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.508973955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509003172Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509032368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509063154Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:30.50910068Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:30.509131826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:30.509162842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_connection connected from Ok(127.0.0.1:38028)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509188909Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_connection connected from Ok(127.0.0.1:54895)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509314753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509348979Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509781085Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509810711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509829359Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.509857926Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.510483238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.510512994Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511052367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511081194Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511099641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511112839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511126418Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:30.511139826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.511206078Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_connection connected from Ok(127.0.0.1:57637)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.511280699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.512384052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.512424517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.512967139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513549347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513585602Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[0]R flush_numbers[0..12]: [284, 280, 284, 284, 285, 284, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513611819Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 172, 172, 172, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513638196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513664853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[1]R flush_numbers[0..12]: [284, 280, 284, 284, 285, 284, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.51368967Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 172, 172, 172, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513719346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513745043Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[2]R flush_numbers[0..12]: [284, 280, 284, 284, 285, 284, 273, 279, 282, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513771219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 172, 172, 172, 165, 169, 170, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513799296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513825213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513850539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:48:40.513878146Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513907662Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513936319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"":"downstairs","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513964745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.513991962Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"0","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514041996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"1","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514071322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"client":"2","":"downstairs","session_id":"14a9887e-0542-4bee-91f1-0032571da053"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514104008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514130545Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"ac324125-2b8c-40a7-97af-809b83204a7d is now active with session: 14a9887e-0542-4bee-91f1-0032571da053","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514155052Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.51416885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878,"session_id":"14a9887e-0542-4bee-91f1-0032571da053"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:40.514182178Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 5 len 4096 data: 13 03/30 Write block 555 len 16384 data: 10 10 12 14 04/30 Read block 422 len 24576 05/30 Read block 86 len 28672 06/30 Write block 589 len 8192 data: 12 12 07/30 Write block 255 len 32768 data: 9 13 13 12 9 8 11 14 08/30 Read block 124 len 12288 09/30 Read block 442 len 4096 10/30 Read block 458 len 16384 11/30 Write block 334 len 40960 data: 8 10 10 13 15 17 15 14 13 13 12/30 Read block 8 len 20480 13/30 Read block 331 len 32768 14/30 Write block 138 len 20480 data: 12 13 9 11 10 15/30 Read block 177 len 40960 16/30 Write block 524 len 36864 data: 12 11 11 11 8 8 7 11 14 17/30 Write block 99 len 32768 data: 7 9 11 11 11 11 13 14 18/30 Read block 299 len 24576 19/30 Read block 2 len 28672 20/30 Read block 177 len 24576 21/30 Write block 50 len 8192 data: 13 15 22/30 Read block 486 len 28672 23/30 Read block 345 len 20480 24/30 Read block 61 len 20480 25/30 Write block 174 len 8192 data: 13 10 26/30 Write block 40 len 4096 data: 16 27/30 Write block 296 len 36864 data: 18 15 12 15 16 11 11 11 11 28/30 Read block 4 len 16384 29/30 Write block 521 len 8192 data: 12 11 30/30 Write block 82 len 24576 data: 11 13 16 16 18 17 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:40.533588357Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 284 284 284 T T T 1 020-039 170 170 170 280 280 280 F F F 2 040-059 172 172 172 284 284 284 T T T 3 060-079 172 172 172 284 284 284 F F F 4 080-099 172 172 172 285 285 285 T T T 5 100-119 172 172 172 284 284 284 T T T 6 120-139 165 165 165 273 273 273 T T T 7 140-159 169 169 169 279 279 279 T T T 8 160-179 170 170 170 282 282 282 T T T 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 169 169 169 279 279 279 T T T 13 260-279 171 171 171 283 283 283 T T T 14 280-299 170 170 170 282 282 282 T T T 15 300-319 172 172 172 285 285 285 T T T 16 320-339 172 172 172 284 284 284 T T T 17 340-359 173 173 173 287 287 287 T T T 18 360-379 173 173 173 287 287 287 F F F 19 380-399 173 173 173 287 287 287 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 172 172 172 284 284 284 T T T 27 540-559 170 170 170 282 282 282 T T T 28 560-579 162 162 162 268 268 268 F F F 29 580-599 166 166 166 276 276 276 T T T Max gen: 173, Max flush: 287 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 170 170 170 280 280 280 F F F 2 040-059 174 174 174 288 288 288 F F F 3 060-079 172 172 172 284 284 284 F F F 4 080-099 174 174 174 288 288 288 F F F 5 100-119 174 174 174 288 288 288 F F F 6 120-139 174 174 174 288 288 288 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 174 174 174 288 288 288 F F F 9 180-199 167 167 167 277 277 277 F F F 10 200-219 170 170 170 282 282 282 F F F 11 220-239 169 169 169 279 279 279 F F F 12 240-259 174 174 174 288 288 288 F F F 13 260-279 174 174 174 288 288 288 F F F 14 280-299 174 174 174 288 288 288 F F F 15 300-319 174 174 174 288 288 288 F F F 16 320-339 174 174 174 288 288 288 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 173 173 173 287 287 287 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 169 169 169 279 279 279 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 174 174 174 288 288 288 F F F 27 540-559 174 174 174 288 288 288 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 174, Max flush: 288 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.832857046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:44.875165579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.875586376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.906726447Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907312574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907349729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907363957Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907379075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} {"msg":"Crucible f9c614a1-88a4-46f1-8ec6-7b6ec11b944c has session id: 7bc1c6d7-e4c2-4205-b035-ae219448ee4d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907787585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Upstairs opts: Upstairs UUID: f9c614a1-88a4-46f1-8ec6-7b6ec11b944c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.90782664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Crucible stats registered with UUID: f9c614a1-88a4-46f1-8ec6-7b6ec11b944c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.907844088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:44.907858476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908369302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908395129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} {"msg":"f9c614a1-88a4-46f1-8ec6-7b6ec11b944c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908457601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908476129Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908498646Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908512925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908526673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908540081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.90855368Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908567088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908582276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908596234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57053)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908771622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60415)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908916044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56379)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.908942711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909524049Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909552645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909572823Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909587341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909600749Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.909620237Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910196255Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910331868Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910525954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.91087912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910908757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910924285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910937173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.910950951Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.91096445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911017993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911038191Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[0]R flush_numbers[0..12]: [288, 280, 288, 284, 288, 288, 288, 288, 288, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911052539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[0]R generation[0..12]: [174, 170, 174, 172, 174, 174, 174, 174, 174, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911070487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911086125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[1]R flush_numbers[0..12]: [288, 280, 288, 284, 288, 288, 288, 288, 288, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911100153Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[1]R generation[0..12]: [174, 170, 174, 172, 174, 174, 174, 174, 174, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911113991Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.9111281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[2]R flush_numbers[0..12]: [288, 280, 288, 284, 288, 288, 288, 288, 288, 277, 282, 279]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911142028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[2]R generation[0..12]: [174, 170, 174, 172, 174, 174, 174, 174, 174, 167, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911155906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911169794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Max found gen is 175","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911182883Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Generation requested: 176 >= found:175","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911196251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911209499Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911228147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"":"downstairs","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911241715Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911253864Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"0","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911269342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"1","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.91128294Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"client":"2","":"downstairs","session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911296349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911308687Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"f9c614a1-88a4-46f1-8ec6-7b6ec11b944c is now active with session: 7bc1c6d7-e4c2-4205-b035-ae219448ee4d","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911321576Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911333764Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886,"session_id":"7bc1c6d7-e4c2-4205-b035-ae219448ee4d"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:44.911346133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 36864 data: 8 10 12 10 12 14 18 19 17 02/30 Read block 177 len 36864 03/30 Write block 123 len 36864 data: 10 12 12 12 15 16 15 17 13 04/30 Read block 467 len 36864 05/30 Write block 279 len 36864 data: 12 9 10 14 12 12 15 11 13 06/30 Write block 429 len 32768 data: 15 10 9 12 12 15 17 15 07/30 Flush 08/30 Write block 27 len 28672 data: 21 19 17 16 16 16 16 09/30 Read block 462 len 16384 10/30 Write block 102 len 16384 data: 12 12 12 14 11/30 Read block 105 len 36864 12/30 Read block 359 len 28672 13/30 Write block 548 len 20480 data: 6 8 9 11 13 14/30 Write block 215 len 4096 data: 12 15/30 Read block 302 len 12288 16/30 Write block 60 len 40960 data: 20 23 20 15 16 15 13 12 15 11 17/30 Write block 105 len 16384 data: 15 15 11 13 18/30 Write block 311 len 8192 data: 12 12 19/30 Flush 20/30 Write block 224 len 32768 data: 15 17 17 16 16 18 15 15 21/30 Read block 448 len 4096 22/30 Write block 329 len 16384 data: 14 13 11 8 23/30 Write block 54 len 24576 data: 12 13 13 16 17 23 24/30 Read block 345 len 8192 25/30 Read block 387 len 32768 26/30 Read block 444 len 36864 27/30 Write block 68 len 32768 data: 16 12 12 14 14 15 14 13 28/30 Read block 283 len 8192 29/30 Write block 397 len 28672 data: 18 19 22 20 19 17 16 30/30 Write block 176 len 32768 data: 17 17 21 20 19 19 16 16 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:44.932106602Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 176 176 176 290 290 290 F F F 2 040-059 174 174 174 288 288 288 T T T 3 060-079 176 176 176 290 290 290 T T T 4 080-099 174 174 174 288 288 288 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 176 176 176 289 289 289 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 174 174 174 288 288 288 T T T 9 180-199 167 167 167 277 277 277 T T T 10 200-219 176 176 176 290 290 290 F F F 11 220-239 169 169 169 279 279 279 T T T 12 240-259 174 174 174 288 288 288 F F F 13 260-279 176 176 176 289 289 289 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 174 174 174 288 288 288 T T T 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 176 176 176 289 289 289 T T T 20 400-419 172 172 172 285 285 285 T T T 21 420-439 176 176 176 289 289 289 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 174 174 174 288 288 288 F F F 27 540-559 176 176 176 290 290 290 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 176, Max flush: 290 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 176 176 176 290 290 290 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 177 177 177 291 291 291 F F F 4 080-099 174 174 174 288 288 288 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 176 176 176 289 289 289 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 177 177 177 291 291 291 F F F 9 180-199 177 177 177 291 291 291 F F F 10 200-219 176 176 176 290 290 290 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 174 174 174 288 288 288 F F F 13 260-279 176 176 176 289 289 289 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 177 177 177 291 291 291 F F F 21 420-439 176 176 176 289 289 289 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 174 174 174 288 288 288 F F F 27 540-559 176 176 176 290 290 290 F F F 28 560-579 162 162 162 268 268 268 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 177, Max flush: 291 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.220104983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:49.262020144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.262417145Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.293158655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.293734323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.293770998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.293785307Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.293801585Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} {"msg":"Crucible 46a154d3-6e1f-4309-826b-11d3dd3df9cb has session id: 60ba2838-7597-4666-b425-b02e498be6cc","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294266257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Upstairs opts: Upstairs UUID: 46a154d3-6e1f-4309-826b-11d3dd3df9cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294308722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Crucible stats registered with UUID: 46a154d3-6e1f-4309-826b-11d3dd3df9cb","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294330269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:49.294345397Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294796051Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294820848Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} {"msg":"46a154d3-6e1f-4309-826b-11d3dd3df9cb active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294875141Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294903728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294928755Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294944173Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294958851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.294973119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295014644Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295041491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295069107Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295094424Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43137)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295120091Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:42125)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295148707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60012)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295261373Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295755922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295799996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.295826733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.29585163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.296059614Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.29609654Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.296564821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.296591058Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.29673444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297146679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297171476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297185874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297264494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.29730302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297328556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297350264Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297374321Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[0]R flush_numbers[0..12]: [288, 290, 291, 291, 288, 290, 289, 288, 291, 291, 290, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297404097Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[0]R generation[0..12]: [174, 176, 177, 177, 174, 176, 176, 174, 177, 177, 176, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297430144Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297454921Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[1]R flush_numbers[0..12]: [288, 290, 291, 291, 288, 290, 289, 288, 291, 291, 290, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297479698Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[1]R generation[0..12]: [174, 176, 177, 177, 174, 176, 176, 174, 177, 177, 176, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297505794Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297533901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[2]R flush_numbers[0..12]: [288, 290, 291, 291, 288, 290, 289, 288, 291, 291, 290, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297560558Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"[2]R generation[0..12]: [174, 176, 177, 177, 174, 176, 176, 174, 177, 177, 176, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297589664Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.29762497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297655276Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297683742Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297711899Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297740635Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"":"downstairs","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297769002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297793929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"0","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297819915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"client":"1","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297847872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"client":"2","":"downstairs","session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297881298Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297899076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"46a154d3-6e1f-4309-826b-11d3dd3df9cb is now active with session: 60ba2838-7597-4666-b425-b02e498be6cc","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297912754Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297924972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894,"session_id":"60ba2838-7597-4666-b425-b02e498be6cc"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:49.297937401Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 4096 data: 13 02/30 Write block 78 len 40960 data: 9 11 11 10 12 14 17 17 19 18 03/30 Write block 172 len 28672 data: 11 11 14 11 18 18 22 04/30 Read block 350 len 12288 05/30 Read block 498 len 8192 06/30 Read block 138 len 8192 07/30 Write block 260 len 32768 data: 9 12 15 14 16 18 15 15 08/30 Read block 222 len 4096 09/30 Read block 568 len 24576 10/30 Write block 209 len 28672 data: 15 17 19 16 16 13 13 11/30 Read block 2 len 40960 12/30 Write block 525 len 36864 data: 12 12 12 9 9 8 12 15 14 13/30 Read block 275 len 24576 14/30 Read block 189 len 28672 15/30 Read block 24 len 24576 16/30 Read block 51 len 20480 17/30 Read block 336 len 4096 18/30 Read block 337 len 20480 19/30 Write block 575 len 20480 data: 11 13 15 10 9 20/30 Read block 494 len 28672 21/30 Write block 70 len 16384 data: 13 15 15 16 22/30 Read block 400 len 24576 23/30 Read block 432 len 20480 24/30 Write block 520 len 4096 data: 10 25/30 Flush 26/30 Flush 27/30 Write block 166 len 12288 data: 13 15 13 28/30 Read block 367 len 28672 29/30 Write block 61 len 12288 data: 24 21 16 30/30 Write block 560 len 8192 data: 15 14 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:49.317564672Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 176 176 176 290 290 290 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 178 178 178 292 292 292 T T T 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 176 176 176 289 289 289 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 178 178 178 292 292 292 T T T 9 180-199 177 177 177 291 291 291 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 178 178 178 292 292 292 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 177 177 177 291 291 291 F F F 21 420-439 176 176 176 289 289 289 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 176 176 176 290 290 290 F F F 28 560-579 178 178 178 292 292 292 T T T 29 580-599 174 174 174 288 288 288 F F F Max gen: 178, Max flush: 292 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 176 176 176 290 290 290 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 179 179 179 293 293 293 F F F 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 176 176 176 289 289 289 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 179 179 179 293 293 293 F F F 9 180-199 177 177 177 291 291 291 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 178 178 178 292 292 292 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 177 177 177 291 291 291 F F F 21 420-439 176 176 176 289 289 289 F F F 22 440-459 173 173 173 287 287 287 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 176 176 176 290 290 290 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 179, Max flush: 293 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.576875557Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.620245498Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.620975587Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.621013872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.621028581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.621043979Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} {"msg":"Crucible 2a57aaa4-0117-40c1-9f2b-438441664070 has session id: ed07cd8b-0637-4ae6-823a-1459f19907ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.621453918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Upstairs opts: Upstairs UUID: 2a57aaa4-0117-40c1-9f2b-438441664070, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.621494163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Crucible stats registered with UUID: 2a57aaa4-0117-40c1-9f2b-438441664070","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62151805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:53.621539617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622024817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622055633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} {"msg":"2a57aaa4-0117-40c1-9f2b-438441664070 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622134753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622154411Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622193416Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622225002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622253929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622305022Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622335088Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622362645Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622391641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622417448Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:52571)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622444875Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:53440)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622474281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:60088)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.622507347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623150667Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623189742Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62328369Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623306827Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623708958Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623824743Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62392808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.623956567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624386504Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62441632Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624432998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624446076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624459774Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624473283Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624502539Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.624988089Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625012136Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[0]R flush_numbers[0..12]: [288, 290, 291, 293, 292, 290, 289, 288, 293, 291, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625027004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[0]R generation[0..12]: [174, 176, 177, 179, 178, 176, 176, 174, 179, 177, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625041482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.6250561Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[1]R flush_numbers[0..12]: [288, 290, 291, 293, 292, 290, 289, 288, 293, 291, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625074568Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[1]R generation[0..12]: [174, 176, 177, 179, 178, 176, 176, 174, 179, 177, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625089056Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625103304Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[2]R flush_numbers[0..12]: [288, 290, 291, 293, 292, 290, 289, 288, 293, 291, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625117473Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[2]R generation[0..12]: [174, 176, 177, 179, 178, 176, 176, 174, 179, 177, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625131491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625145699Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625158877Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625172346Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625185534Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:48:53.625198732Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"":"downstairs","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62521767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625231778Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"0","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625246186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"1","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625259915Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"client":"2","":"downstairs","session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625277113Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625290371Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"2a57aaa4-0117-40c1-9f2b-438441664070 is now active with session: ed07cd8b-0637-4ae6-823a-1459f19907ca","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.62530347Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625315918Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902,"session_id":"ed07cd8b-0637-4ae6-823a-1459f19907ca"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:53.625329266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 28672 data: 20 18 17 14 11 12 10 02/30 Read block 449 len 4096 03/30 Flush 04/30 Write block 542 len 4096 data: 12 05/30 Read block 459 len 24576 06/30 Read block 530 len 16384 07/30 Read block 596 len 8192 08/30 Write block 25 len 20480 data: 22 21 22 20 18 09/30 Read block 199 len 40960 10/30 Read block 409 len 4096 11/30 Write block 129 len 36864 data: 16 18 14 13 14 11 14 11 11 12/30 Read block 44 len 8192 13/30 Read block 471 len 12288 14/30 Write block 184 len 24576 data: 15 14 14 13 15 13 15/30 Read block 285 len 8192 16/30 Write block 409 len 36864 data: 11 11 11 10 13 14 17 15 17 17/30 Flush 18/30 Write block 424 len 20480 data: 21 20 19 17 18 19/30 Read block 108 len 16384 20/30 Read block 307 len 28672 21/30 Read block 314 len 24576 22/30 Read block 387 len 24576 23/30 Write block 441 len 16384 data: 15 18 19 18 24/30 Read block 305 len 16384 25/30 Read block 162 len 16384 26/30 Read block 567 len 24576 27/30 Read block 12 len 36864 28/30 Read block 388 len 40960 29/30 Write block 448 len 8192 data: 11 10 30/30 Write block 67 len 36864 data: 13 17 13 14 16 16 17 15 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:53.645530176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 180 180 180 295 295 295 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 179 179 179 293 293 293 T T T 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 180 180 180 295 295 295 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 179 179 179 293 293 293 F F F 9 180-199 180 180 180 295 295 295 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 178 178 178 292 292 292 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 180 180 180 295 295 295 F F F 21 420-439 176 176 176 289 289 289 T T T 22 440-459 173 173 173 287 287 287 T T T 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 180 180 180 295 295 295 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 180, Max flush: 295 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 288 288 288 F F F 1 020-039 180 180 180 295 295 295 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 181 181 181 296 296 296 F F F 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 180 180 180 295 295 295 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 179 179 179 293 293 293 F F F 9 180-199 180 180 180 295 295 295 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 178 178 178 292 292 292 F F F 14 280-299 176 176 176 289 289 289 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 173 173 173 287 287 287 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 180 180 180 295 295 295 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 181 181 181 296 296 296 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 180 180 180 295 295 295 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 174 174 174 288 288 288 F F F Max gen: 181, Max flush: 296 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.876687249Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:57.919298464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.919689526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.951311676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.951937938Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.951972444Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.951986092Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.95200069Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} {"msg":"Crucible 2caa8c61-9f52-42bd-8a19-314ad860bf72 has session id: e860cda6-d208-4731-abf1-cedd9b9deb21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.95248763Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Upstairs opts: Upstairs UUID: 2caa8c61-9f52-42bd-8a19-314ad860bf72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.952530684Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Crucible stats registered with UUID: 2caa8c61-9f52-42bd-8a19-314ad860bf72","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.952555611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:57.952577688Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.952980189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953008115Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} {"msg":"2caa8c61-9f52-42bd-8a19-314ad860bf72 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953071217Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953090925Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953118821Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953137729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953163366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953191622Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953219269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953245685Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953272892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953298549Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:36778)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953325826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:44452)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953357462Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:35196)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.953386238Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954033458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954073382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954103469Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954130055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954216675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954247371Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.95465924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.954785544Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955096885Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.95537758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955401977Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955416006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955475468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955495326Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955509894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955648707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955667924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[0]R flush_numbers[0..12]: [288, 295, 291, 296, 292, 290, 295, 288, 293, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955683033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[0]R generation[0..12]: [174, 180, 177, 181, 178, 176, 180, 174, 179, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955697251Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955714969Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[1]R flush_numbers[0..12]: [288, 295, 291, 296, 292, 290, 295, 288, 293, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955729207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[1]R generation[0..12]: [174, 180, 177, 181, 178, 176, 180, 174, 179, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955743105Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955757023Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[2]R flush_numbers[0..12]: [288, 295, 291, 296, 292, 290, 295, 288, 293, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955770772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[2]R generation[0..12]: [174, 180, 177, 181, 178, 176, 180, 174, 179, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.95578434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955798338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955811437Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955824665Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955837593Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955850632Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"":"downstairs","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955871099Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955883518Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"0","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955896866Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"1","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955910234Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"client":"2","":"downstairs","session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955926492Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955939351Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"2caa8c61-9f52-42bd-8a19-314ad860bf72 is now active with session: e860cda6-d208-4731-abf1-cedd9b9deb21","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955952009Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955964098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910,"session_id":"e860cda6-d208-4731-abf1-cedd9b9deb21"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:48:57.955976396Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 36864 data: 23 23 22 23 21 19 17 17 17 02/30 Flush 03/30 Write block 367 len 20480 data: 15 13 14 14 14 04/30 Write block 278 len 12288 data: 12 13 10 05/30 Write block 67 len 36864 data: 14 18 14 15 17 17 18 16 15 06/30 Read block 398 len 28672 07/30 Read block 77 len 40960 08/30 Flush 09/30 Write block 63 len 32768 data: 17 17 16 14 15 19 15 16 10/30 Write block 591 len 4096 data: 13 11/30 Read block 32 len 36864 12/30 Read block 249 len 20480 13/30 Write block 138 len 4096 data: 13 14/30 Read block 208 len 8192 15/30 Read block 471 len 20480 16/30 Flush 17/30 Write block 455 len 8192 data: 12 13 18/30 Write block 7 len 40960 data: 17 18 18 18 18 19 19 24 22 20 19/30 Read block 385 len 36864 20/30 Read block 237 len 12288 21/30 Flush 22/30 Write block 166 len 40960 data: 14 16 14 12 10 10 12 12 15 12 23/30 Write block 63 len 20480 data: 18 18 17 15 16 24/30 Read block 508 len 40960 25/30 Read block 407 len 24576 26/30 Read block 344 len 32768 27/30 Read block 343 len 8192 28/30 Read block 121 len 36864 29/30 Write block 401 len 28672 data: 21 19 18 15 12 13 11 30/30 Read block 147 len 40960 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 7 9 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:48:57.976736396Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 182 182 182 297 297 297 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 182 182 182 299 299 299 T T T 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 179 179 179 293 293 293 T T T 9 180-199 180 180 180 295 295 295 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 182 182 182 298 298 298 F F F 14 280-299 182 182 182 298 298 298 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 182 182 182 298 298 298 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 180 180 180 295 295 295 T T T 21 420-439 181 181 181 296 296 296 F F F 22 440-459 182 182 182 300 300 300 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 180 180 180 295 295 295 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 182, Max flush: 300 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 F F F 1 020-039 182 182 182 297 297 297 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 183 183 183 301 301 301 F F F 4 080-099 178 178 178 292 292 292 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 174 174 174 288 288 288 F F F 8 160-179 183 183 183 301 301 301 F F F 9 180-199 180 180 180 295 295 295 F F F 10 200-219 178 178 178 292 292 292 F F F 11 220-239 177 177 177 291 291 291 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 182 182 182 298 298 298 F F F 14 280-299 182 182 182 298 298 298 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 182 182 182 298 298 298 F F F 19 380-399 177 177 177 291 291 291 F F F 20 400-419 183 183 183 301 301 301 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 182 182 182 300 300 300 F F F 23 460-479 173 173 173 287 287 287 F F F 24 480-499 172 172 172 286 286 286 F F F 25 500-519 172 172 172 284 284 284 F F F 26 520-539 178 178 178 292 292 292 F F F 27 540-559 180 180 180 295 295 295 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 183, Max flush: 301 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.237936127Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.281534079Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282139014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.2821711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282186098Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282201526Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} {"msg":"Crucible a30c6e13-ef22-4121-aad0-7c32205d8fda has session id: 6cdead82-4339-4a39-ab69-eba574b42ee4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282687336Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Upstairs opts: Upstairs UUID: a30c6e13-ef22-4121-aad0-7c32205d8fda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282735Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Crucible stats registered with UUID: a30c6e13-ef22-4121-aad0-7c32205d8fda","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.282762947Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:02.282787953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283246707Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283273973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} {"msg":"a30c6e13-ef22-4121-aad0-7c32205d8fda active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283352563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283371611Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283394438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283409636Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283428574Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283443112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.28345748Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283471579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283489516Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283524002Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54277)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.28361565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52503)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283656485Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52587)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.283685142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.284283438Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.284312434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.284420881Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.284458406Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.284915579Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.2850714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285433205Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.28547224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285501446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285523184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285537572Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.28555007Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285563619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285577037Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.285977267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286445929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286471676Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[0]R flush_numbers[0..12]: [300, 297, 291, 301, 292, 290, 299, 288, 301, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286486614Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[0]R generation[0..12]: [182, 182, 177, 183, 178, 176, 182, 174, 183, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286500842Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.28651517Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[1]R flush_numbers[0..12]: [300, 297, 291, 301, 292, 290, 299, 288, 301, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286529388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[1]R generation[0..12]: [182, 182, 177, 183, 178, 176, 182, 174, 183, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286547216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286561904Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[2]R flush_numbers[0..12]: [300, 297, 291, 301, 292, 290, 299, 288, 301, 295, 292, 291]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286575923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[2]R generation[0..12]: [182, 182, 177, 183, 178, 176, 182, 174, 183, 180, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286589721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286603619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286616838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286630166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286643204Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:49:02.286656143Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"":"downstairs","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286674381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286686409Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"0","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286699857Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"1","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286713216Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"client":"2","":"downstairs","session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286726584Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286739013Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"a30c6e13-ef22-4121-aad0-7c32205d8fda is now active with session: 6cdead82-4339-4a39-ab69-eba574b42ee4","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286753931Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286766459Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918,"session_id":"6cdead82-4339-4a39-ab69-eba574b42ee4"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:02.286778817Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 399 len 32768 data: 23 21 22 20 19 16 13 14 02/30 Read block 522 len 4096 03/30 Read block 313 len 20480 04/30 Write block 440 len 8192 data: 16 16 05/30 Read block 283 len 32768 06/30 Read block 267 len 4096 07/30 Read block 473 len 40960 08/30 Write block 86 len 16384 data: 20 19 18 19 09/30 Read block 401 len 28672 10/30 Write block 461 len 16384 data: 14 15 12 13 11/30 Write block 445 len 4096 data: 14 12/30 Read block 405 len 40960 13/30 Write block 538 len 12288 data: 14 14 13 14/30 Read block 284 len 8192 15/30 Read block 466 len 16384 16/30 Write block 217 len 24576 data: 14 12 8 12 12 13 17/30 Read block 69 len 24576 18/30 Read block 286 len 12288 19/30 Read block 394 len 16384 20/30 Flush 21/30 Write block 498 len 40960 data: 19 21 20 16 17 19 19 17 17 16 22/30 Write block 175 len 20480 data: 13 19 19 23 21 23/30 Write block 460 len 28672 data: 17 15 16 13 14 17 18 24/30 Flush 25/30 Write block 187 len 24576 data: 14 16 14 11 12 11 26/30 Flush 27/30 Write block 152 len 40960 data: 21 22 21 20 22 20 14 10 12 14 28/30 Write block 3 len 12288 data: 13 12 14 29/30 Write block 365 len 28672 data: 11 15 16 14 15 15 15 30/30 Read block 324 len 24576 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 2 4 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:02.306049053Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 300 300 300 T T T 1 020-039 182 182 182 297 297 297 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 183 183 183 301 301 301 F F F 4 080-099 184 184 184 302 302 302 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 174 174 174 288 288 288 T T T 8 160-179 184 184 184 303 303 303 T T T 9 180-199 184 184 184 304 304 304 F F F 10 200-219 184 184 184 302 302 302 F F F 11 220-239 184 184 184 302 302 302 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 182 182 182 298 298 298 F F F 14 280-299 182 182 182 298 298 298 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 182 182 182 298 298 298 T T T 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 184 184 184 302 302 302 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 184 184 184 303 303 303 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 184 184 184 302 302 302 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 184, Max flush: 304 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 182 182 182 297 297 297 F F F 2 040-059 177 177 177 291 291 291 F F F 3 060-079 183 183 183 301 301 301 F F F 4 080-099 184 184 184 302 302 302 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 185 185 185 305 305 305 F F F 9 180-199 184 184 184 304 304 304 F F F 10 200-219 184 184 184 302 302 302 F F F 11 220-239 184 184 184 302 302 302 F F F 12 240-259 178 178 178 292 292 292 F F F 13 260-279 182 182 182 298 298 298 F F F 14 280-299 182 182 182 298 298 298 F F F 15 300-319 176 176 176 290 290 290 F F F 16 320-339 177 177 177 291 291 291 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 185 185 185 305 305 305 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 184 184 184 302 302 302 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 184 184 184 303 303 303 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 184 184 184 302 302 302 F F F 28 560-579 179 179 179 293 293 293 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 185, Max flush: 305 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.541966046Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:06.583889666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.58425721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.614937457Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.615521055Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61555826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.615577678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.615593556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} {"msg":"Crucible 1b23ce7f-4f95-4e04-a448-9d40c3593cb7 has session id: 58c3bff3-59e4-4d85-b1be-1f00f61a06e2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616033341Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Upstairs opts: Upstairs UUID: 1b23ce7f-4f95-4e04-a448-9d40c3593cb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616077606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Crucible stats registered with UUID: 1b23ce7f-4f95-4e04-a448-9d40c3593cb7","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616103283Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:06.61612328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61652983Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616554317Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} {"msg":"1b23ce7f-4f95-4e04-a448-9d40c3593cb7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616618729Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616647765Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616671702Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61668722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616729285Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616761541Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616794187Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616822423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61685225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616882196Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37074)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616911112Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41447)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.616941679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62032)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61701032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617668688Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617710143Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617726081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617808111Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617832548Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.617848466Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.618395198Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.618426244Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.618446292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.618989744Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619027199Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619054856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619076323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619091301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.61910514Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619117708Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619131456Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[0]R flush_numbers[0..12]: [305, 297, 291, 301, 302, 290, 299, 305, 305, 304, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619145745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[0]R generation[0..12]: [185, 182, 177, 183, 184, 176, 182, 185, 185, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619160103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619174451Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[1]R flush_numbers[0..12]: [305, 297, 291, 301, 302, 290, 299, 305, 305, 304, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619188389Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[1]R generation[0..12]: [185, 182, 177, 183, 184, 176, 182, 185, 185, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619202258Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619219935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[2]R flush_numbers[0..12]: [305, 297, 291, 301, 302, 290, 299, 305, 305, 304, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619240953Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[2]R generation[0..12]: [185, 182, 177, 183, 184, 176, 182, 185, 185, 184, 184, 184]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:49:06.619267719Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619294816Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619312464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619326332Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619339641Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619352939Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"":"downstairs","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619366267Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619378566Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"0","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619392114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"1","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619405472Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"client":"2","":"downstairs","session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619418781Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619431349Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"1b23ce7f-4f95-4e04-a448-9d40c3593cb7 is now active with session: 58c3bff3-59e4-4d85-b1be-1f00f61a06e2","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619444297Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619463565Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926,"session_id":"58c3bff3-59e4-4d85-b1be-1f00f61a06e2"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:06.619477433Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 229 len 32768 data: 19 16 16 14 14 15 15 16 02/30 Write block 178 len 28672 data: 24 22 20 20 17 17 16 03/30 Write block 363 len 24576 data: 11 14 12 16 17 15 04/30 Read block 319 len 28672 05/30 Read block 12 len 8192 06/30 Read block 374 len 16384 07/30 Read block 485 len 20480 08/30 Write block 573 len 4096 data: 12 09/30 Read block 190 len 16384 10/30 Write block 313 len 40960 data: 17 13 12 12 13 13 13 15 13 15 11/30 Read block 281 len 8192 12/30 Flush 13/30 Write block 274 len 40960 data: 9 12 11 10 13 14 11 11 15 13 14/30 Flush 15/30 Write block 564 len 20480 data: 13 8 6 7 9 16/30 Write block 360 len 36864 data: 11 8 11 12 15 13 17 18 16 17/30 Write block 510 len 28672 data: 17 15 16 16 13 12 10 18/30 Write block 60 len 28672 data: 21 25 22 19 19 18 16 19/30 Write block 238 len 28672 data: 11 11 10 10 10 6 6 20/30 Write block 206 len 4096 data: 13 21/30 Read block 322 len 12288 22/30 Write block 289 len 4096 data: 14 23/30 Write block 48 len 36864 data: 13 11 14 16 12 14 13 14 14 24/30 Flush 25/30 Write block 161 len 32768 data: 15 11 11 16 15 15 17 15 26/30 Write block 59 len 28672 data: 24 22 26 23 20 20 19 27/30 Write block 302 len 40960 data: 12 12 12 10 11 11 10 11 12 13 28/30 Read block 83 len 24576 29/30 Write block 30 len 40960 data: 18 18 18 17 17 16 17 13 14 16 30/30 Read block 572 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 5 6 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:06.639788679Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 182 182 182 297 297 297 T T T 2 040-059 186 186 186 308 308 308 T T T 3 060-079 186 186 186 308 308 308 T T T 4 080-099 184 184 184 302 302 302 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 186 186 186 306 306 306 T T T 9 180-199 186 186 186 306 306 306 F F F 10 200-219 186 186 186 308 308 308 F F F 11 220-239 186 186 186 308 308 308 F F F 12 240-259 186 186 186 308 308 308 F F F 13 260-279 186 186 186 307 307 307 F F F 14 280-299 186 186 186 308 308 308 F F F 15 300-319 186 186 186 306 306 306 T T T 16 320-339 186 186 186 306 306 306 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 184 184 184 302 302 302 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 186 186 186 308 308 308 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 184 184 184 302 302 302 F F F 28 560-579 186 186 186 308 308 308 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 186, Max flush: 308 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 187 187 187 309 309 309 F F F 2 040-059 187 187 187 309 309 309 F F F 3 060-079 187 187 187 309 309 309 F F F 4 080-099 184 184 184 302 302 302 F F F 5 100-119 176 176 176 290 290 290 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 187 187 187 309 309 309 F F F 9 180-199 186 186 186 306 306 306 F F F 10 200-219 186 186 186 308 308 308 F F F 11 220-239 186 186 186 308 308 308 F F F 12 240-259 186 186 186 308 308 308 F F F 13 260-279 186 186 186 307 307 307 F F F 14 280-299 186 186 186 308 308 308 F F F 15 300-319 187 187 187 309 309 309 F F F 16 320-339 186 186 186 306 306 306 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 181 181 181 296 296 296 F F F 22 440-459 184 184 184 302 302 302 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 186 186 186 308 308 308 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 184 184 184 302 302 302 F F F 28 560-579 186 186 186 308 308 308 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 187, Max flush: 309 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.919193377Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:10.961884392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.962338745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.993426872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994172669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994216004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994239791Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994258309Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} {"msg":"Crucible 1586e2ed-3fee-4eeb-bd7f-759859ad6e3e has session id: 8b83d945-075d-428c-a804-e041ba26d4f5","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994784673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Upstairs opts: Upstairs UUID: 1586e2ed-3fee-4eeb-bd7f-759859ad6e3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994820019Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Crucible stats registered with UUID: 1586e2ed-3fee-4eeb-bd7f-759859ad6e3e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.994835577Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:10.994848796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995336265Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995360892Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} {"msg":"1586e2ed-3fee-4eeb-bd7f-759859ad6e3e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995423464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995442682Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995465289Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995479907Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995496515Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995521222Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995553968Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995580595Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995611001Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995640407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40330)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995669494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41589)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.99570032Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49833)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.995729396Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996483722Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996514059Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996530207Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996544655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996558493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.996615396Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997164328Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997306Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997393789Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997685213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997741176Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997774932Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997905186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997928313Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997942481Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997955119Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997968678Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[0]R flush_numbers[0..12]: [305, 309, 309, 309, 302, 290, 299, 305, 309, 306, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997982756Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[0]R generation[0..12]: [185, 187, 187, 187, 184, 176, 182, 185, 187, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.997997034Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998011563Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[1]R flush_numbers[0..12]: [305, 309, 309, 309, 302, 290, 299, 305, 309, 306, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998025861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[1]R generation[0..12]: [185, 187, 187, 187, 184, 176, 182, 185, 187, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998039649Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998057137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[2]R flush_numbers[0..12]: [305, 309, 309, 309, 302, 290, 299, 305, 309, 306, 308, 308]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998071345Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[2]R generation[0..12]: [185, 187, 187, 187, 184, 176, 182, 185, 187, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998085343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998099302Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.99811229Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998125598Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998138777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998151905Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"":"downstairs","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998165133Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998177282Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"0","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.99819063Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"1","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:49:10.998203759Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"client":"2","":"downstairs","session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998220897Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998233445Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"1586e2ed-3fee-4eeb-bd7f-759859ad6e3e is now active with session: 8b83d945-075d-428c-a804-e041ba26d4f5","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998246343Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.998258522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934,"session_id":"8b83d945-075d-428c-a804-e041ba26d4f5"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:10.99827399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 118 len 4096 data: 14 02/30 Flush 03/30 Write block 98 len 8192 data: 9 8 04/30 Flush 05/30 Write block 220 len 40960 data: 13 13 14 14 16 18 18 17 17 20 06/30 Read block 19 len 8192 07/30 Read block 235 len 4096 08/30 Write block 38 len 28672 data: 15 17 17 14 15 16 14 09/30 Read block 489 len 8192 10/30 Read block 159 len 32768 11/30 Write block 432 len 36864 data: 13 13 16 18 16 13 16 19 17 12/30 Write block 299 len 28672 data: 16 17 12 13 13 13 11 13/30 Read block 203 len 24576 14/30 Read block 136 len 4096 15/30 Read block 391 len 36864 16/30 Read block 558 len 24576 17/30 Write block 205 len 20480 data: 14 14 12 14 16 18/30 Write block 217 len 4096 data: 15 19/30 Read block 562 len 24576 20/30 Read block 223 len 24576 21/30 Read block 371 len 16384 22/30 Flush 23/30 Write block 257 len 20480 data: 14 13 10 10 13 24/30 Read block 135 len 28672 25/30 Write block 34 len 20480 data: 18 17 18 14 16 26/30 Write block 559 len 16384 data: 17 16 15 15 27/30 Write block 288 len 8192 data: 18 15 28/30 Read block 446 len 28672 29/30 Write block 238 len 24576 data: 12 12 11 11 11 7 30/30 Write block 211 len 8192 data: 20 17 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:11.018285733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 188 188 188 312 312 312 T T T 2 040-059 188 188 188 312 312 312 F F F 3 060-079 187 187 187 309 309 309 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 187 187 187 309 309 309 F F F 9 180-199 186 186 186 306 306 306 F F F 10 200-219 188 188 188 312 312 312 T T T 11 220-239 188 188 188 312 312 312 T T T 12 240-259 186 186 186 308 308 308 T T T 13 260-279 186 186 186 307 307 307 T T T 14 280-299 188 188 188 312 312 312 T T T 15 300-319 188 188 188 312 312 312 F F F 16 320-339 186 186 186 306 306 306 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 188 188 188 312 312 312 F F F 22 440-459 188 188 188 312 312 312 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 186 186 186 308 308 308 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 184 184 184 302 302 302 T T T 28 560-579 186 186 186 308 308 308 T T T 29 580-599 182 182 182 299 299 299 F F F Max gen: 188, Max flush: 312 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 189 189 189 313 313 313 F F F 2 040-059 188 188 188 312 312 312 F F F 3 060-079 187 187 187 309 309 309 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 187 187 187 309 309 309 F F F 9 180-199 186 186 186 306 306 306 F F F 10 200-219 189 189 189 313 313 313 F F F 11 220-239 189 189 189 313 313 313 F F F 12 240-259 189 189 189 313 313 313 F F F 13 260-279 189 189 189 313 313 313 F F F 14 280-299 189 189 189 313 313 313 F F F 15 300-319 188 188 188 312 312 312 F F F 16 320-339 186 186 186 306 306 306 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 188 188 188 312 312 312 F F F 22 440-459 188 188 188 312 312 312 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 F F F 25 500-519 186 186 186 308 308 308 F F F 26 520-539 184 184 184 302 302 302 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 189 189 189 313 313 313 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 189, Max flush: 313 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.311027954Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.354585232Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355245139Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355280065Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355294583Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355313851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} {"msg":"Crucible c4fde9fb-a890-48d5-8419-baef507ffe8a has session id: e5d079ac-ffce-463b-b2b0-10b8f1232a53","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355716511Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Upstairs opts: Upstairs UUID: c4fde9fb-a890-48d5-8419-baef507ffe8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355759426Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Crucible stats registered with UUID: c4fde9fb-a890-48d5-8419-baef507ffe8a","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.355784123Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:15.35580619Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356294319Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356330465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} {"msg":"c4fde9fb-a890-48d5-8419-baef507ffe8a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356361301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356375109Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356397896Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356412694Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356426733Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356440021Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356465668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356480286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356557147Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356590872Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:58896)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356681331Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:42978)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356722616Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:44532)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.356789398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.357359537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.357387373Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.357480252Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.357524996Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.357929186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358044802Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358482767Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358511104Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358526262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.35853906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358564627Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358591774Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358866029Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.358894906Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.359445137Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.359940986Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.359961753Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[0]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 309, 306, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.359978601Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[0]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 187, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.35999292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360018476Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[1]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 309, 306, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360034874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[1]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 187, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360048742Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360062711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[2]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 309, 306, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360076669Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[2]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 187, 186, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360094027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360108615Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360121844Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360135182Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.36014822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360161208Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"":"downstairs","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360174477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:49:15.360186865Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"0","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360205493Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"1","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360218721Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"client":"2","":"downstairs","session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.36023214Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360244648Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"c4fde9fb-a890-48d5-8419-baef507ffe8a is now active with session: e5d079ac-ffce-463b-b2b0-10b8f1232a53","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360257736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360269935Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942,"session_id":"e5d079ac-ffce-463b-b2b0-10b8f1232a53"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:15.360282464Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 272 len 20480 data: 6 9 10 13 12 02/30 Read block 369 len 36864 03/30 Read block 428 len 20480 04/30 Write block 485 len 28672 data: 13 14 15 14 14 14 10 05/30 Read block 546 len 28672 06/30 Write block 513 len 16384 data: 17 14 13 11 07/30 Read block 441 len 40960 08/30 Read block 422 len 24576 09/30 Write block 175 len 40960 data: 14 20 20 25 23 21 21 18 18 17 10/30 Read block 259 len 40960 11/30 Read block 190 len 24576 12/30 Write block 332 len 24576 data: 9 7 9 11 11 14 13/30 Read block 104 len 24576 14/30 Read block 253 len 28672 15/30 Read block 441 len 12288 16/30 Read block 179 len 8192 17/30 Read block 313 len 20480 18/30 Write block 564 len 24576 data: 14 9 7 8 10 8 19/30 Read block 240 len 20480 20/30 Write block 377 len 8192 data: 12 13 21/30 Write block 214 len 20480 data: 14 14 15 16 13 22/30 Read block 200 len 8192 23/30 Read block 156 len 8192 24/30 Read block 455 len 32768 25/30 Read block 557 len 36864 26/30 Write block 420 len 36864 data: 13 16 16 19 22 21 20 18 19 27/30 Read block 195 len 8192 28/30 Read block 366 len 12288 29/30 Write block 316 len 16384 data: 13 14 14 14 30/30 Write block 525 len 40960 data: 13 13 13 10 10 9 13 16 15 14 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:15.381526553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 189 189 189 313 313 313 F F F 2 040-059 188 188 188 312 312 312 F F F 3 060-079 187 187 187 309 309 309 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 187 187 187 309 309 309 T T T 9 180-199 186 186 186 306 306 306 T T T 10 200-219 189 189 189 313 313 313 T T T 11 220-239 189 189 189 313 313 313 F F F 12 240-259 189 189 189 313 313 313 F F F 13 260-279 189 189 189 313 313 313 T T T 14 280-299 189 189 189 313 313 313 F F F 15 300-319 188 188 188 312 312 312 T T T 16 320-339 186 186 186 306 306 306 T T T 17 340-359 174 174 174 288 288 288 F F F 18 360-379 186 186 186 308 308 308 T T T 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 188 188 188 312 312 312 T T T 22 440-459 188 188 188 312 312 312 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 184 184 184 303 303 303 T T T 25 500-519 186 186 186 308 308 308 T T T 26 520-539 184 184 184 302 302 302 T T T 27 540-559 189 189 189 313 313 313 F F F 28 560-579 189 189 189 313 313 313 T T T 29 580-599 182 182 182 299 299 299 F F F Max gen: 189, Max flush: 313 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 189 189 189 313 313 313 F F F 2 040-059 188 188 188 312 312 312 F F F 3 060-079 187 187 187 309 309 309 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 190 190 190 314 314 314 F F F 10 200-219 190 190 190 314 314 314 F F F 11 220-239 189 189 189 313 313 313 F F F 12 240-259 189 189 189 313 313 313 F F F 13 260-279 190 190 190 314 314 314 F F F 14 280-299 189 189 189 313 313 313 F F F 15 300-319 190 190 190 314 314 314 F F F 16 320-339 190 190 190 314 314 314 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 190 190 190 314 314 314 F F F 19 380-399 184 184 184 302 302 302 F F F 20 400-419 184 184 184 302 302 302 F F F 21 420-439 190 190 190 314 314 314 F F F 22 440-459 188 188 188 312 312 312 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 190 190 190 314 314 314 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 190 190 190 314 314 314 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 190, Max flush: 314 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.661422301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.704600425Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705301458Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705353491Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705383338Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705400606Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} {"msg":"Crucible 9efaae1f-af30-4a3f-b5df-6763d14ac173 has session id: e1a58989-a385-4e62-9b36-d89de03623ef","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705833352Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Upstairs opts: Upstairs UUID: 9efaae1f-af30-4a3f-b5df-6763d14ac173, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705867468Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Crucible stats registered with UUID: 9efaae1f-af30-4a3f-b5df-6763d14ac173","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.705884376Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:29.705898204Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706432028Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706468163Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} {"msg":"9efaae1f-af30-4a3f-b5df-6763d14ac173 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706527556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706564691Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706603076Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706633412Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706661269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706690675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706722731Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706747268Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706774035Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706803621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:56926)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706832628Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:64921)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706857695Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:59042)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.706881132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.707397808Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.707427184Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.707532211Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.707552528Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.707999243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708140465Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708205757Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708228824Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708529907Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708560213Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708576241Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708759398Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708779366Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708793404Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.708866325Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709356934Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709385081Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[0]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709401008Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[0]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709415737Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709430125Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[1]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709444323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[1]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709458261Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709475959Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[2]R flush_numbers[0..12]: [305, 313, 312, 309, 311, 310, 299, 305, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709490477Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[2]R generation[0..12]: [185, 189, 188, 187, 188, 188, 182, 185, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709504586Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709518894Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709532132Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.70954581Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-11T23:49:29.709559299Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709575407Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"":"downstairs","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709589555Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709601923Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"0","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709615822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"1","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.70962963Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"client":"2","":"downstairs","session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709643269Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709655937Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"9efaae1f-af30-4a3f-b5df-6763d14ac173 is now active with session: e1a58989-a385-4e62-9b36-d89de03623ef","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709669085Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709681344Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950,"session_id":"e1a58989-a385-4e62-9b36-d89de03623ef"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:29.709696142Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 16384 data: 24 24 24 23 02/30 Flush 03/30 Flush 04/30 Write block 421 len 12288 data: 17 17 20 05/30 Read block 96 len 4096 06/30 Read block 80 len 28672 07/30 Write block 391 len 28672 data: 13 11 13 15 19 20 19 08/30 Read block 22 len 8192 09/30 Write block 484 len 24576 data: 14 14 15 16 15 15 10/30 Read block 409 len 4096 11/30 Write block 51 len 4096 data: 17 12/30 Read block 305 len 4096 13/30 Read block 63 len 28672 14/30 Write block 65 len 4096 data: 20 15/30 Read block 143 len 24576 16/30 Write block 261 len 28672 data: 14 16 15 17 19 16 16 17/30 Read block 73 len 28672 18/30 Write block 33 len 32768 data: 18 19 18 19 15 17 18 18 19/30 Write block 440 len 28672 data: 18 17 19 20 19 15 15 20/30 Write block 26 len 4096 data: 24 21/30 Write block 273 len 32768 data: 10 11 14 13 11 14 15 12 22/30 Write block 490 len 8192 data: 15 11 23/30 Write block 306 len 16384 data: 12 12 11 12 24/30 Read block 274 len 36864 25/30 Read block 244 len 20480 26/30 Read block 527 len 28672 27/30 Write block 145 len 40960 data: 16 13 12 14 16 15 16 22 23 22 28/30 Read block 49 len 20480 29/30 Write block 418 len 16384 data: 14 14 14 18 30/30 Read block 249 len 12288 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 3 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 27 26 25 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:29.728352073Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 192 192 192 315 315 315 T T T 2 040-059 188 188 188 312 312 312 T T T 3 060-079 187 187 187 309 309 309 T T T 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 185 185 185 305 305 305 T T T 8 160-179 190 190 190 314 314 314 F F F 9 180-199 190 190 190 314 314 314 F F F 10 200-219 190 190 190 314 314 314 F F F 11 220-239 189 189 189 313 313 313 F F F 12 240-259 189 189 189 313 313 313 F F F 13 260-279 190 190 190 314 314 314 T T T 14 280-299 189 189 189 313 313 313 T T T 15 300-319 190 190 190 314 314 314 T T T 16 320-339 190 190 190 314 314 314 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 190 190 190 314 314 314 F F F 19 380-399 184 184 184 302 302 302 T T T 20 400-419 184 184 184 302 302 302 T T T 21 420-439 190 190 190 314 314 314 T T T 22 440-459 188 188 188 312 312 312 T T T 23 460-479 184 184 184 303 303 303 F F F 24 480-499 190 190 190 314 314 314 T T T 25 500-519 190 190 190 314 314 314 F F F 26 520-539 190 190 190 314 314 314 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 192, Max flush: 315 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 193 193 193 316 316 316 F F F 2 040-059 193 193 193 316 316 316 F F F 3 060-079 193 193 193 316 316 316 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 190 190 190 314 314 314 F F F 10 200-219 190 190 190 314 314 314 F F F 11 220-239 189 189 189 313 313 313 F F F 12 240-259 189 189 189 313 313 313 F F F 13 260-279 193 193 193 316 316 316 F F F 14 280-299 193 193 193 316 316 316 F F F 15 300-319 193 193 193 316 316 316 F F F 16 320-339 190 190 190 314 314 314 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 190 190 190 314 314 314 F F F 19 380-399 193 193 193 316 316 316 F F F 20 400-419 193 193 193 316 316 316 F F F 21 420-439 193 193 193 316 316 316 F F F 22 440-459 193 193 193 316 316 316 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 193 193 193 316 316 316 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 190 190 190 314 314 314 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 182 182 182 299 299 299 F F F Max gen: 193, Max flush: 316 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.010036128Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.053939972Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.054804435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.054860268Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.054883895Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.054909042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} {"msg":"Crucible 9c81898e-e689-498d-bb8d-e7ae1875d45c has session id: 1e4f1e6e-0012-4267-b8d6-e7e22eddef3e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.055434537Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Upstairs opts: Upstairs UUID: 9c81898e-e689-498d-bb8d-e7ae1875d45c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.055478421Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Crucible stats registered with UUID: 9c81898e-e689-498d-bb8d-e7ae1875d45c","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.055503758Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:34.055525675Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.055996257Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056034202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} {"msg":"9c81898e-e689-498d-bb8d-e7ae1875d45c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056106603Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056125831Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056148668Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056163066Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056181174Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056194922Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.05621582Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056240497Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056262874Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.05629006Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47667)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056319987Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45374)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056349993Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48401)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.056379189Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.057228064Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.05725901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.057273798Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.057287747Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.057301985Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.057316553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058035803Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058076998Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058108845Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058711879Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058740266Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058756004Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058828075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058862201Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058878709Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058891617Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058905455Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[0]R flush_numbers[0..12]: [305, 316, 316, 316, 311, 310, 299, 316, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058919524Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[0]R generation[0..12]: [185, 193, 193, 193, 188, 188, 182, 193, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058933822Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.0589482Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[1]R flush_numbers[0..12]: [305, 316, 316, 316, 311, 310, 299, 316, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058962408Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[1]R generation[0..12]: [185, 193, 193, 193, 188, 188, 182, 193, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058980446Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.058997014Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[2]R flush_numbers[0..12]: [305, 316, 316, 316, 311, 310, 299, 316, 314, 314, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059022861Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[2]R generation[0..12]: [185, 193, 193, 193, 188, 188, 182, 193, 190, 190, 190, 189]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059042158Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2024-12-11T23:49:34.059068955Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059103711Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059120929Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059134277Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059147605Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"":"downstairs","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059160924Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059173102Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"0","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059186651Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"1","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059200149Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"client":"2","":"downstairs","session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059213647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059226155Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"9c81898e-e689-498d-bb8d-e7ae1875d45c is now active with session: 1e4f1e6e-0012-4267-b8d6-e7e22eddef3e","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059242114Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059254522Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958,"session_id":"1e4f1e6e-0012-4267-b8d6-e7e22eddef3e"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:34.059266901Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 4096 data: 16 02/30 Write block 592 len 28672 data: 12 12 12 10 7 7 6 03/30 Write block 203 len 24576 data: 12 12 15 15 13 15 04/30 Read block 136 len 8192 05/30 Write block 378 len 24576 data: 14 13 13 12 10 11 06/30 Read block 441 len 40960 07/30 Read block 527 len 40960 08/30 Read block 27 len 16384 09/30 Flush 10/30 Write block 520 len 40960 data: 11 13 12 13 13 14 14 14 11 11 11/30 Read block 224 len 24576 12/30 Write block 189 len 4096 data: 15 13/30 Flush 14/30 Write block 22 len 16384 data: 23 25 25 25 15/30 Write block 231 len 12288 data: 17 15 15 16/30 Write block 36 len 40960 data: 20 16 18 19 19 15 16 17 15 13 17/30 Write block 25 len 16384 data: 26 25 24 22 18/30 Read block 113 len 20480 19/30 Write block 314 len 32768 data: 14 13 14 15 15 15 16 14 20/30 Read block 233 len 36864 21/30 Write block 455 len 16384 data: 13 14 13 10 22/30 Write block 492 len 16384 data: 14 17 17 20 23/30 Read block 303 len 4096 24/30 Flush 25/30 Write block 127 len 4096 data: 16 26/30 Read block 141 len 8192 27/30 Write block 244 len 16384 data: 7 6 6 9 28/30 Read block 335 len 4096 29/30 Read block 470 len 24576 30/30 Write block 408 len 20480 data: 10 12 12 12 11 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:34.077805777Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 194 194 194 319 319 319 F F F 3 060-079 193 193 193 316 316 316 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 182 182 182 299 299 299 T T T 7 140-159 193 193 193 316 316 316 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 317 317 317 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 189 189 189 313 313 313 T T T 13 260-279 193 193 193 316 316 316 F F F 14 280-299 194 194 194 317 317 317 F F F 15 300-319 194 194 194 319 319 319 F F F 16 320-339 194 194 194 319 319 319 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 194 194 194 317 317 317 F F F 19 380-399 194 194 194 317 317 317 F F F 20 400-419 193 193 193 316 316 316 T T T 21 420-439 193 193 193 316 316 316 F F F 22 440-459 194 194 194 319 319 319 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 194 194 194 319 319 319 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 194 194 194 318 318 318 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 194 194 194 317 317 317 F F F Max gen: 194, Max flush: 319 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 194 194 194 319 319 319 F F F 3 060-079 193 193 193 316 316 316 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 195 195 195 320 320 320 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 317 317 317 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 195 195 195 320 320 320 F F F 13 260-279 193 193 193 316 316 316 F F F 14 280-299 194 194 194 317 317 317 F F F 15 300-319 194 194 194 319 319 319 F F F 16 320-339 194 194 194 319 319 319 F F F 17 340-359 174 174 174 288 288 288 F F F 18 360-379 194 194 194 317 317 317 F F F 19 380-399 194 194 194 317 317 317 F F F 20 400-419 195 195 195 320 320 320 F F F 21 420-439 193 193 193 316 316 316 F F F 22 440-459 194 194 194 319 319 319 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 194 194 194 319 319 319 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 194 194 194 318 318 318 F F F 27 540-559 189 189 189 313 313 313 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 194 194 194 317 317 317 F F F Max gen: 195, Max flush: 320 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.305915219Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.349904103Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.350533655Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35057086Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.350591248Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.350606946Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} {"msg":"Crucible e28891c4-e87f-4c1e-a5e8-b1bdb1682386 has session id: ad044eec-857c-4c97-8478-ae31b2f16eb0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351118202Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Upstairs opts: Upstairs UUID: e28891c4-e87f-4c1e-a5e8-b1bdb1682386, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351160917Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Crucible stats registered with UUID: e28891c4-e87f-4c1e-a5e8-b1bdb1682386","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351185714Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:38.351206011Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35170061Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351806567Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} {"msg":"e28891c4-e87f-4c1e-a5e8-b1bdb1682386 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351838463Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35186254Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351889966Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351918293Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351946379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.351972486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352003292Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352029839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352054666Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352080862Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48468)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352107889Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33248)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352130486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55961)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352155973Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352698826Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352726772Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35274247Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352756329Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35282423Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.352859796Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353337956Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353373322Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353402988Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353926553Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35395634Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353972367Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353985486Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.353999224Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354012853Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354025301Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354038779Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[0]R flush_numbers[0..12]: [305, 319, 319, 316, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354052788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[0]R generation[0..12]: [185, 194, 194, 193, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354067206Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354081524Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[1]R flush_numbers[0..12]: [305, 319, 319, 316, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354095372Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[1]R generation[0..12]: [185, 194, 194, 193, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354109281Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354123399Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[2]R flush_numbers[0..12]: [305, 319, 319, 316, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354141026Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[2]R generation[0..12]: [185, 194, 194, 193, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354155505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354169633Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354182851Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Generation requested: 196 >= found:196","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354203559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354228116Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354250243Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"":"downstairs","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.35427531Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354293327Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"0","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354310496Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"client":"1","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354335262Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1966,"client":"2","":"downstairs","session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.3543554Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354368348Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"e28891c4-e87f-4c1e-a5e8-b1bdb1682386 is now active with session: ad044eec-857c-4c97-8478-ae31b2f16eb0","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354381487Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354393805Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966,"session_id":"ad044eec-857c-4c97-8478-ae31b2f16eb0"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:38.354409033Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 416 len 20480 data: 16 18 15 15 15 02/30 Write block 534 len 32768 data: 15 17 17 14 15 15 14 13 03/30 Write block 346 len 28672 data: 13 11 11 11 12 11 11 04/30 Read block 149 len 24576 05/30 Read block 480 len 8192 06/30 Read block 201 len 12288 07/30 Read block 336 len 40960 08/30 Read block 174 len 28672 09/30 Write block 495 len 20480 data: 21 21 20 20 22 10/30 Read block 366 len 32768 11/30 Read block 450 len 4096 12/30 Read block 560 len 36864 13/30 Read block 262 len 4096 14/30 Read block 233 len 32768 15/30 Read block 544 len 4096 16/30 Write block 44 len 32768 data: 16 14 13 12 14 12 15 18 17/30 Read block 412 len 32768 18/30 Flush 19/30 Write block 529 len 20480 data: 12 10 14 17 16 20/30 Read block 148 len 16384 21/30 Flush 22/30 Write block 63 len 24576 data: 21 21 21 17 17 20 23/30 Flush 24/30 Flush 25/30 Write block 458 len 4096 data: 11 26/30 Read block 23 len 24576 27/30 Read block 343 len 16384 28/30 Read block 565 len 20480 29/30 Read block 56 len 40960 30/30 Read block 398 len 20480 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:38.37447186Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 196 196 196 321 321 321 F F F 3 060-079 196 196 196 323 323 323 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 195 195 195 320 320 320 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 317 317 317 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 195 195 195 320 320 320 F F F 13 260-279 193 193 193 316 316 316 F F F 14 280-299 194 194 194 317 317 317 F F F 15 300-319 194 194 194 319 319 319 F F F 16 320-339 194 194 194 319 319 319 F F F 17 340-359 196 196 196 321 321 321 F F F 18 360-379 194 194 194 317 317 317 F F F 19 380-399 194 194 194 317 317 317 F F F 20 400-419 196 196 196 321 321 321 F F F 21 420-439 196 196 196 321 321 321 F F F 22 440-459 194 194 194 319 319 319 T T T 23 460-479 184 184 184 303 303 303 F F F 24 480-499 196 196 196 321 321 321 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 196 196 196 322 322 322 F F F 27 540-559 196 196 196 321 321 321 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 194 194 194 317 317 317 F F F Max gen: 196, Max flush: 323 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 196 196 196 321 321 321 F F F 3 060-079 196 196 196 323 323 323 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 188 188 188 310 310 310 F F F 6 120-139 195 195 195 320 320 320 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 190 190 190 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 194 194 194 317 317 317 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 195 195 195 320 320 320 F F F 13 260-279 193 193 193 316 316 316 F F F 14 280-299 194 194 194 317 317 317 F F F 15 300-319 194 194 194 319 319 319 F F F 16 320-339 194 194 194 319 319 319 F F F 17 340-359 196 196 196 321 321 321 F F F 18 360-379 194 194 194 317 317 317 F F F 19 380-399 194 194 194 317 317 317 F F F 20 400-419 196 196 196 321 321 321 F F F 21 420-439 196 196 196 321 321 321 F F F 22 440-459 197 197 197 324 324 324 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 196 196 196 321 321 321 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 196 196 196 322 322 322 F F F 27 540-559 196 196 196 321 321 321 F F F 28 560-579 190 190 190 314 314 314 F F F 29 580-599 194 194 194 317 317 317 F F F Max gen: 197, Max flush: 324 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.5905342Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:42.632834093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.633202378Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.664169379Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.664793392Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d81f7fe7d3a87a2a3c58f4f5d619f7f37b0a05d0\",\n git_commit_timestamp: \"2024-12-11T23:24:14.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.664829447Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.664844075Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.664864723Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} {"msg":"Crucible 8c230d61-52d3-4da4-bfe0-0c0e9b6103d5 has session id: 38cabf95-7f8e-479b-aecc-a73e33bb22af","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665298849Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Upstairs opts: Upstairs UUID: 8c230d61-52d3-4da4-bfe0-0c0e9b6103d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665332745Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Crucible stats registered with UUID: 8c230d61-52d3-4da4-bfe0-0c0e9b6103d5","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665349253Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:42.665362621Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665809286Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665835082Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} {"msg":"8c230d61-52d3-4da4-bfe0-0c0e9b6103d5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665897225Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665916042Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.66593871Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665953788Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665969736Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.665994323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666020559Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666044316Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666076382Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666105179Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_connection connected from Ok(127.0.0.1:34144)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666134365Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_connection connected from Ok(127.0.0.1:54922)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666163171Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_connection connected from Ok(127.0.0.1:33764)","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666305494Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666874933Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666917898Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.666952044Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.66698337Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.667011576Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.667031984Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.667614461Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.667657556Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.667692981Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668232074Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.66826508Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668281728Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668294697Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668309505Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668323093Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668335592Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.66834948Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[0]R flush_numbers[0..12]: [305, 319, 321, 323, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668363838Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[0]R generation[0..12]: [185, 194, 196, 196, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668378166Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668392434Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[1]R flush_numbers[0..12]: [305, 319, 321, 323, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668406323Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[1]R generation[0..12]: [185, 194, 196, 196, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668420291Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668434509Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[2]R flush_numbers[0..12]: [305, 319, 321, 323, 311, 310, 320, 316, 314, 318, 317, 319]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668448647Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[2]R generation[0..12]: [185, 194, 196, 196, 188, 188, 195, 193, 190, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668465025Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668479314Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668492542Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668506031Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668525388Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668538856Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"":"downstairs","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668569912Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668584271Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"0","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668597839Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"1","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668611027Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"client":"2","":"downstairs","session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668624296Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668636704Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"8c230d61-52d3-4da4-bfe0-0c0e9b6103d5 is now active with session: 38cabf95-7f8e-479b-aecc-a73e33bb22af","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668649673Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.668661891Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974,"session_id":"38cabf95-7f8e-479b-aecc-a73e33bb22af"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-12-11T23:49:42.66867435Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 12288 data: 13 16 12 02/30 Flush 03/30 Write block 560 len 36864 data: 17 16 16 13 15 10 8 9 11 04/30 Read block 190 len 40960 05/30 Write block 353 len 40960 data: 10 9 12 12 8 9 11 12 9 12 06/30 Read block 383 len 36864 07/30 Write block 278 len 4096 data: 15 08/30 Write block 109 len 16384 data: 13 14 17 20 09/30 Write block 219 len 4096 data: 9 10/30 Write block 589 len 40960 data: 13 13 14 13 13 13 11 8 8 7 11/30 Read block 306 len 8192 12/30 Write block 586 len 32768 data: 8 11 11 14 14 15 14 14 13/30 Read block 42 len 32768 14/30 Read block 57 len 16384 15/30 Flush 16/30 Write block 389 len 40960 data: 9 11 14 12 14 16 20 21 20 20 17/30 Flush 18/30 Write block 354 len 20480 data: 10 13 13 9 10 19/30 Flush 20/30 Write block 278 len 24576 data: 16 16 13 12 17 14 21/30 Read block 488 len 16384 22/30 Read block 537 len 8192 23/30 Write block 56 len 36864 data: 15 17 18 25 23 27 24 22 22 24/30 Write block 160 len 24576 data: 14 17 13 12 17 16 25/30 Read block 422 len 24576 26/30 Read block 136 len 16384 27/30 Read block 269 len 40960 28/30 Read block 199 len 4096 29/30 Read block 418 len 32768 30/30 Write block 331 len 8192 data: 12 10 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-11T23:49:42.689264381Z","hostname":"w-01JEW1EDBDZJ4HH98YP2BSQWYV","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 196 196 196 321 321 321 T T T 3 060-079 196 196 196 323 323 323 T T T 4 080-099 188 188 188 311 311 311 F F F 5 100-119 198 198 198 326 326 326 F F F 6 120-139 195 195 195 320 320 320 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 198 198 198 325 325 325 T T T 9 180-199 194 194 194 318 318 318 F F F 10 200-219 198 198 198 326 326 326 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 195 195 195 320 320 320 F F F 13 260-279 198 198 198 326 326 326 T T T 14 280-299 194 194 194 317 317 317 T T T 15 300-319 194 194 194 319 319 319 F F F 16 320-339 194 194 194 319 319 319 T T T 17 340-359 198 198 198 328 328 328 F F F 18 360-379 198 198 198 326 326 326 F F F 19 380-399 198 198 198 327 327 327 F F F 20 400-419 196 196 196 321 321 321 F F F 21 420-439 196 196 196 321 321 321 F F F 22 440-459 197 197 197 324 324 324 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 196 196 196 321 321 321 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 196 196 196 322 322 322 F F F 27 540-559 196 196 196 321 321 321 F F F 28 560-579 198 198 198 326 326 326 F F F 29 580-599 198 198 198 326 326 326 F F F Max gen: 198, Max flush: 328 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 305 305 305 F F F 1 020-039 194 194 194 319 319 319 F F F 2 040-059 199 199 199 329 329 329 F F F 3 060-079 199 199 199 329 329 329 F F F 4 080-099 188 188 188 311 311 311 F F F 5 100-119 198 198 198 326 326 326 F F F 6 120-139 195 195 195 320 320 320 F F F 7 140-159 193 193 193 316 316 316 F F F 8 160-179 199 199 199 329 329 329 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 198 198 198 326 326 326 F F F 11 220-239 194 194 194 319 319 319 F F F 12 240-259 195 195 195 320 320 320 F F F 13 260-279 199 199 199 329 329 329 F F F 14 280-299 199 199 199 329 329 329 F F F 15 300-319 194 194 194 319 319 319 F F F 16 320-339 199 199 199 329 329 329 F F F 17 340-359 198 198 198 328 328 328 F F F 18 360-379 198 198 198 326 326 326 F F F 19 380-399 198 198 198 327 327 327 F F F 20 400-419 196 196 196 321 321 321 F F F 21 420-439 196 196 196 321 321 321 F F F 22 440-459 197 197 197 324 324 324 F F F 23 460-479 184 184 184 303 303 303 F F F 24 480-499 196 196 196 321 321 321 F F F 25 500-519 190 190 190 314 314 314 F F F 26 520-539 196 196 196 322 322 322 F F F 27 540-559 196 196 196 321 321 321 F F F 28 560-579 198 198 198 326 326 326 F F F 29 580-599 198 198 198 326 326 326 F F F Max gen: 199, Max flush: 329 9:24 Test duration Test completed real 9:24.640459375 user 50.020018947 sys 18.824442260 trap 0.073860060 tflt 0.047000807 dflt 0.104338802 kflt 0.006420204 lock 15:08:25.387695798 slp 46:10.013948843 lat 5.608310074 stop 0.061338264