/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.322381671Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"e5f834981d3a43d75fe6f427c4755541ac0c2270\",\n git_commit_timestamp: \"2024-07-09T21:07:43.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.322982921Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323016128Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984} {"msg":"Crucible 1864b505-57b3-46f5-afb7-3f893d1c8ce4 has session id: 1fa1968f-e6cc-483d-a5ce-934a36e16807","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323141785Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Upstairs opts: Upstairs UUID: 1864b505-57b3-46f5-afb7-3f893d1c8ce4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.32319184Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Crucible stats registered with UUID: 1864b505-57b3-46f5-afb7-3f893d1c8ce4","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323219148Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323542596Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984} {"msg":"1864b505-57b3-46f5-afb7-3f893d1c8ce4 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323567763Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323583342Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.32359853Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323612529Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323626397Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323639946Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323653535Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323670633Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323690491Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323788591Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_connection connected from Ok(127.0.0.1:35343)","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.323887672Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-09T22:02:46.324077922Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-09T22:02:46.32410774Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-09T22:02:46.324125598Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_connection connected from Ok(127.0.0.1:56554)","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324148226Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324179543Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.32420447Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324231307Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324739517Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324767234Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324782543Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.324803961Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.325449287Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.325477384Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.325978864Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.326006062Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.32602146Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.326034739Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.326048297Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:46.326061636Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.325658501Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_connection connected from Ok(127.0.0.1:41035)","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.326019315Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.326903928Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.326940234Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.327534595Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328103849Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328131616Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[0]R flush_numbers[0..12]: [370, 385, 393, 384, 394, 396, 396, 391, 357, 392, 392, 390]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328153364Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[0]R generation[0..12]: [187, 194, 198, 193, 198, 198, 198, 196, 180, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328168422Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[0]R dirty[0..12]: [false, true, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328182981Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[1]R flush_numbers[0..12]: [370, 385, 393, 384, 394, 396, 396, 391, 357, 392, 392, 390]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328196949Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[1]R generation[0..12]: [187, 194, 198, 193, 198, 198, 198, 196, 180, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328210798Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[1]R dirty[0..12]: [false, true, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328226526Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[2]R flush_numbers[0..12]: [370, 385, 393, 384, 394, 396, 396, 391, 357, 392, 392, 390]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328240765Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[2]R generation[0..12]: [187, 194, 198, 193, 198, 198, 198, 196, 180, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328254824Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"[2]R dirty[0..12]: [false, true, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328269022Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328282271Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.32829573Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Next flush: 397","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328308838Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328321867Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328335776Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"extent:1 gens: 194 194 194","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328352614Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"extent:1 flush: 385 385 385 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328367923Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328382821Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"extent:1 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.32839726Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328411908Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328426617Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328441085Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328456214Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328469662Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328483031Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"starting reconciliation d861ef6b-f32f-4c52-a3b5-d545fe0c640c: found 1 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.3284973Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328511698Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 397, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.328543705Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.330728729Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.330761665Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344716202Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"":"downstairs","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344744629Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"0","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-09T22:02:56.344760098Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"1","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344776256Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"client":"2","":"downstairs","session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344790145Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344802564Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"1864b505-57b3-46f5-afb7-3f893d1c8ce4 is now active with session: 1fa1968f-e6cc-483d-a5ce-934a36e16807","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344815822Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984,"session_id":"1fa1968f-e6cc-483d-a5ce-934a36e16807"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-07-09T22:02:56.344828141Z","hostname":"w-01J2CNYPQGXQVK3WVQV8ZVEK6G","pid":1984} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Read 88 Done Done Done false 2 Acked 1001 Read 256 Done Done Done false 3 Acked 1002 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 3 2 1 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 4 3 2 1 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program