/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.45397161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454217431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937} {"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454239023Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937} {"msg":"Crucible bf888960-046f-4ce1-b2f2-59b7afbb3784 has session id: 181a0233-db86-4db5-bdbc-f715b22eb9e1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454255278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Upstairs opts: Upstairs UUID: bf888960-046f-4ce1-b2f2-59b7afbb3784, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454284717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"Crucible runtime is spawned 181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Crucible stats registered with UUID: bf888960-046f-4ce1-b2f2-59b7afbb3784","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454324002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454376953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937} {"msg":"bf888960-046f-4ce1-b2f2-59b7afbb3784 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454494191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454515333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454532977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454549711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454571363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454600542Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454625673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454652683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.45468949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454720259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:46.454754206Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:46.454802099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454832908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:46.454863887Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454888858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454914408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454939229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.454970858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.455309115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.455361526Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.455389116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.455417376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.455909138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456049417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456346809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456410046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456439475Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456492856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456517147Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.456532212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.456125473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.456201266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.456917666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.456937659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457405299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457862743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457904638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[0]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 351, 351, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457933368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[0]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 198, 198, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457956459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457974862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[1]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 351, 351, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.457992596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[1]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 198, 198, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45800993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458027843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[2]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 350, 340, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458045367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[2]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 197, 187, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458062921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, true, true, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458080734Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458097128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458113802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Next flush: 353","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458134075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 5 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458150869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458168362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:5 gens: 197 197 197","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458186606Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:5 flush: 350 350 350 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458205299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458223912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:5 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458241756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458259999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458278103Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458296016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 7 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458314299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458331593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:7 gens: 198 198 197","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458349647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:7 flush: 351 351 350 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45836825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458388332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:7 dirty: false false true","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458421001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458438094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458452869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458467174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 8 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458481879Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458495844Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:8 gens: 198 198 187","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458510128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:8 flush: 351 351 340 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458524813Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458539278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:8 dirty: false false true","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458553693Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458568427Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458582962Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458597077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 9 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458613911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458627886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:9 gens: 188 188 188","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458642211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:9 flush: 341 341 341 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458657615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458672369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:9 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458686834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458701509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458715904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458730359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 16 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458744593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458758368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:16 gens: 198 198 191","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458772793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:16 flush: 352 352 343 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458787658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458802202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:16 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458817967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458832691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458846896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458861251Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 17 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458875506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 17","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458889241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:17 gens: 192 192 192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458903626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:17 flush: 344 344 344 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45891861Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458933305Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:17 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45894777Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 17","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458962314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458977359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.458991944Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 21 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459006388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 21","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459021903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:21 gens: 198 198 196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459036457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:21 flush: 351 351 349 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459051392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459065917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:21 dirty: false false true","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459080272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459095096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 21","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459109621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459123836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 22 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459138081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459151836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:22 gens: 198 198 192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45916603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:22 flush: 351 351 344 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459180795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45919556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:22 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459212414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459227318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459241733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459256068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extents 29 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459270203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 29","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459283958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:29 gens: 198 198 191","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459298253Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:29 flush: 352 352 343 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459313037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459327552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:29 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459342017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 29","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459356531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459370956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459385181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extent 27 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459399756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 27","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45941513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:27 gens: 198 198 196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459429725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:27 flush: 351 351 348 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459444589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459459244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:27 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459473739Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459488413Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 27","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459503168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459517583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Extent 28 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459532238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"First source client ID for extent 28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459546013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:28 gens: 198 198 194","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459560387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:28 flush: 351 351 345 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459575302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459590007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"extent:28 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459604562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459620725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"find dest for source 0 for extent at index 28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45963549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459650015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45966502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459678425Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.45969178Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Found 11 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459705255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Full repair list: {5: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 8: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 9: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 16: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 21: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 22: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 7: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 29: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 17: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 27: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 28: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459729866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 5, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 5, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 8, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 8, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 9, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 9, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 16, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 16, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 21, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 21, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 22, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 22, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 7, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 7, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 29, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 29, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 17, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 17, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 27, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 27, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 28, client_id: ClientId(0), flush_number: 353, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 28, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 28 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459875503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459890458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459904013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.459918568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.462488615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.462522173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.462540286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.513882563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.513914841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.513930436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.514492712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.514519342Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.514534396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.552971354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.553003023Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.553059912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.554955384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.554983304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.555009915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.595717368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.595746168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.595761472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.5974207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.59744817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.597463535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638185184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638220181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638239734Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638728856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638758166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.638784516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.67781118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.6778394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.677859562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.679363867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.679391177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.67940923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.720480202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.720509672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.720525256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.720997355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.721024255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.72103947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.758188474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.758221602Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.758240145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.759593674Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.759629161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.75966088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.797588412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.797617702Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.797632996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.799035038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.799063637Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.799087868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.838593038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.838626186Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.838653067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.839172928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.839204367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.839230238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.876516552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.876555168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.876574192Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.877050109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"no action needed request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.877078509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"sending reconcile request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.877094623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914115344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"":"downstairs","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914144883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"0","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914160198Z","hostname":"Wait for a query_work_queue command to finish before sending IO ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"1","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914178011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"client":"2","":"downstairs","session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914191896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914204342Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"bf888960-046f-4ce1-b2f2-59b7afbb3784 is now active with session: 181a0233-db86-4db5-bdbc-f715b22eb9e1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914218217Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937,"session_id":"181a0233-db86-4db5-bdbc-f715b22eb9e1"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:47.914231272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1937} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:2 States: Active Active Active GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY 6 Acked 1005 Read 100 Done Done Done false 7 Acked 1006 Read 100 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:2 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program