/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.714563709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.714950782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.714971839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979} {"msg":"Crucible a38b9e5c-95fa-448f-8bcc-11227d0f2bbc has session id: 418c243e-cdd4-4e7b-a4c6-0da53068f505","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715188237Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Upstairs opts: Upstairs UUID: a38b9e5c-95fa-448f-8bcc-11227d0f2bbc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.71523175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Crucible stats registered with UUID: a38b9e5c-95fa-448f-8bcc-11227d0f2bbc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715255327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715538565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979} {"msg":"a38b9e5c-95fa-448f-8bcc-11227d0f2bbc active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715644319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715662757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715677735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715691642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.71570546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715718668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715732456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715762962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715789858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.715807045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_connection connected from Ok(127.0.0.1:39489)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716002696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:30.71604434Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:30.716077725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:30.7161116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.71617716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716192508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_connection connected from Ok(127.0.0.1:47881)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716206336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716220344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.71678818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716814656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716829054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.716842892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.717477148Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.71759913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.717959026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.717991512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.718006049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.718144899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.718163396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.718176584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71719167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_connection connected from Ok(127.0.0.1:53222)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.717249002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.718282258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.718333511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.718792683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719392094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71941381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[0]R flush_numbers[0..12]: [393, 404, 404, 403, 403, 389, 403, 395, 403, 403, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719434227Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[0]R generation[0..12]: [192, 198, 198, 198, 198, 190, 198, 194, 198, 198, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719449385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[0]R dirty[0..12]: [true, true, false, false, false, false, true, false, false, false, true, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719463783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[1]R flush_numbers[0..12]: [393, 404, 404, 403, 403, 389, 403, 395, 403, 403, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719478071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[1]R generation[0..12]: [192, 198, 198, 198, 198, 190, 198, 194, 198, 198, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719491699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[1]R dirty[0..12]: [true, true, false, false, false, false, true, false, false, false, true, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719505947Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[2]R flush_numbers[0..12]: [393, 404, 404, 403, 403, 389, 403, 395, 403, 403, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719519555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[2]R generation[0..12]: [192, 198, 198, 198, 198, 190, 198, 194, 198, 198, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719533023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"[2]R dirty[0..12]: [true, true, false, false, false, false, true, false, false, false, true, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719546811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719560459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719573437Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Next flush: 405","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719586085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719598893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719612391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:0 gens: 192 192 192","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719629059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:0 flush: 393 393 393 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719644016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719658364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:0 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719672352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71968658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719700538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719714586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719729044Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719742662Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:1 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71975666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:1 flush: 404 404 404 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719771397Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719785786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:1 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719799643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719814431Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719830899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719845347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Extents 6 dirty","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719859364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719873822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:6 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71988832Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:6 flush: 403 403 403 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719903068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719917346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:6 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719931424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"find dest for source 0 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719945652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.71995968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719973678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Extents 10 dirty","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.719987745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"First source client ID for extent 10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720016811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:10 gens: 197 197 197","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720031999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:10 flush: 402 402 402 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720046807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720063374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"extent:10 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720077582Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"find dest for source 0 for extent at index 10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.72009189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720105958Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720119806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720134254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720147472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.72016061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"starting reconciliation b3aeb6aa-b18b-4091-8ed5-e98e10e63baf: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720174448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(6): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(10): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720191975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 405, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 405, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(6), client_id: ClientId(0), flush_number: 405, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(6), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(10), client_id: ClientId(0), flush_number: 405, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(10) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(10), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(10) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.720260235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.721767102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.721794678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.735553118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.735581964Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.74755393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.747582495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.759319335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.759348241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770450816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"":"downstairs","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770478852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"0","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770494159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"1","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770511037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"client":"2","":"downstairs","session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770525085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770537673Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"a38b9e5c-95fa-448f-8bcc-11227d0f2bbc is now active with session: 418c243e-cdd4-4e7b-a4c6-0da53068f505","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770551291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979,"session_id":"418c243e-cdd4-4e7b-a4c6-0da53068f505"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:40.770563329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Read 88 Done Done Done false 2 Acked 1001 Read 256 Done Done Done false 3 Acked 1002 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 3 2 1 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 4 3 2 1 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:44.820218383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1979}