/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.702373488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.702807336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946} {"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.702842482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946} {"msg":"Crucible d8c6b890-4eed-4ce1-8cef-f2d474a9472e has session id: 8e6e8ec5-811b-42e0-b68a-cbc4762a0b02","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.704601775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Upstairs opts: Upstairs UUID: d8c6b890-4eed-4ce1-8cef-f2d474a9472e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.704631711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Crucible stats registered with UUID: d8c6b890-4eed-4ce1-8cef-f2d474a9472e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70464595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.704927547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946} {"msg":"d8c6b890-4eed-4ce1-8cef-f2d474a9472e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705039214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705058041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705078859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705092927Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705106986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705125263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70515586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705178207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705196885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705217262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705340928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.705379693Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70540955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706072302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706099949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706114857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706128805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706152462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706181279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706633706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.706662902Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70668262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707181301Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707209658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707224276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707237335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707250903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707264471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70727671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707290378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[0]R flush_numbers[0..12]: [356, 345, 323, 345, 343, 339, 357, 351, 348, 351, 355, 355]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707305207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[0]R generation[0..12]: [198, 194, 179, 194, 192, 190, 198, 197, 195, 197, 198, 198]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707319545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, false, false, true]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707333753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[1]R flush_numbers[0..12]: [356, 345, 323, 345, 343, 339, 357, 351, 348, 351, 355, 355]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707347871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[1]R generation[0..12]: [198, 194, 179, 194, 192, 190, 198, 197, 195, 197, 198, 198]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70736185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, false, false, true]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707379818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[2]R flush_numbers[0..12]: [349, 345, 323, 345, 343, 339, 351, 351, 348, 351, 351, 351]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707394116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[2]R generation[0..12]: [196, 194, 179, 194, 192, 190, 197, 197, 195, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707407895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707422103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707435151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70744858Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707461418Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extents 5 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707474417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707488305Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:5 gens: 190 190 190","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707502723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:5 flush: 339 339 339 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707517991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70753263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:5 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707546928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707561656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707578084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707592583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extents 11 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70761657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 11","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707642357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:11 gens: 198 198 197","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707665764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:11 flush: 355 355 351 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707693911Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707710449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:11 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707724847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 11","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707739356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707753654Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707767882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extents 26 dirty","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70778761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707813917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:26 gens: 194 194 194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707843553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:26 flush: 346 346 346 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707880029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707912525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:26 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707943411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707977427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.707994695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708009274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70803809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708052708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:0 gens: 198 198 196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708067317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:0 flush: 356 356 349 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708082315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708096873Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708111262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70812649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708145058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708159806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708174364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708188143Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:6 gens: 198 198 197","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708202571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:6 flush: 357 357 351 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708217499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708232057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:6 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708246676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708261764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708276422Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708290851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 10 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708305369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 10","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708319137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:10 gens: 198 198 197","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708335275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:10 flush: 355 355 351 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708350304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708364892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:10 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70837938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708393978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 10","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708408817Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708423235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 12 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708437833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708451481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:12 gens: 198 198 190","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70846593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:12 flush: 352 352 338 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708480958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708495676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:12 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708510165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708524933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708541221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708555739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 13 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708570287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 13","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708584006Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:13 gens: 198 198 194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708598394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:13 flush: 352 352 346 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708613322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708627971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:13 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708642529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708657337Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 13","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708671876Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708686294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 14 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708700952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708714741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:14 gens: 198 198 196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708731219Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:14 flush: 354 354 350 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708746167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708760985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:14 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708775553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708790352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70880506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708819658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 15 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708834167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 15","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708848095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:15 gens: 198 198 196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708862613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:15 flush: 352 352 350 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708877521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.7088922Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:15 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708906898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708923286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 15","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708938034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708952382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Extent 17 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708967721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"First source client ID for extent 17","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708981389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:17 gens: 198 198 197","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.708995748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:17 flush: 354 354 351 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709010566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709025274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"extent:17 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709039762Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709054481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"find dest for source 0 for extent at index 17","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709069099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709083517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709098425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709111854Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709126832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Found 11 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.70914008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Full repair list: {6: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 12: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 11: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 13: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 26: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 17: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 0: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 14: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 5: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 10: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, 15: ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709163928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 6, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 6, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 12, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 12, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 11, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 11, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 13, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 13, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 26, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 26, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 17, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 17, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 0, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 0, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 14, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 14, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 5, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 5, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 10, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 10, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 15, client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 15 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 15, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 15 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709305371Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709320479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709333968Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709347246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709360514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709373853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.709387141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.761285698Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.761325113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.761348061Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.761895676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.761929942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.76194596Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.803244566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.803288061Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.803323057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.804942146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.804970802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.804986501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.848437932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.848479527Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.848498675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.849026963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.84905406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.849073197Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.88895091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.888982236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.888998405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.890755798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.890790723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.89082268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.935143429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.935266025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.935309939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.93573009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.935763906Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.935783414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.976279054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.97630974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.976329558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.976833068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.976862105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.976887002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016206561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016236057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016252445Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016754766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016782903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.016797981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.056166994Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.056202629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.056239845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.057774174Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.05780757Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.057830538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.097610472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.097655836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.097675894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.098289342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.098317369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.098332407Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.141143584Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.14118001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.141224974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.141722166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"no action needed request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.141749413Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"sending reconcile request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.141766851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.18192395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"":"downstairs","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.181980054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"0","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.181999412Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"1","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.182023399Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"client":"2","":"downstairs","session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.182039187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.182051995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"d8c6b890-4eed-4ce1-8cef-f2d474a9472e is now active with session: 8e6e8ec5-811b-42e0-b68a-cbc4762a0b02","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.182065164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946,"session_id":"8e6e8ec5-811b-42e0-b68a-cbc4762a0b02"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:18.182077802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1946} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY 4 Acked 1003 Read 100 Done Done Done false 5 Acked 1004 Read 100 Done Done Done false 6 Acked 1005 Read 100 Done Done Done false 7 Acked 1006 Read 100 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:4 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program