{"msg":"Opened existing region file \"/var/tmp/test_up/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.390609968Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.390908644Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.39093107Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.391094517Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Blocks per extent:10 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.391120243Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\nCommit timestamp: 2024-02-28T21:51:33.000000000Z branch: main\nrustc: 1.75.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.392059585Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.392101629Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_up/8820\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.396393699Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.398646489Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.399018374Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.39904517Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:55.399066277Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"accepted connection from 127.0.0.1:48627","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.472402643Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from c3ffd5d2-c5f0-4d65-a4d5-70967df67e1f with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.473894234Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: c3ffd5d2-c5f0-4d65-a4d5-70967df67e1f, session_id: a9be0b57-4663-4b9a-b7eb-c2303d240fbe, gen: 1 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.473930819Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: c3ffd5d2-c5f0-4d65-a4d5-70967df67e1f, session_id: a9be0b57-4663-4b9a-b7eb-c2303d240fbe, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.475823111Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.47719795Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:00.477297805Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: c3ffd5d2-c5f0-4d65-a4d5-70967df67e1f, session_id: a9be0b57-4663-4b9a-b7eb-c2303d240fbe, gen: 1 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:04.493473872Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: c3ffd5d2-c5f0-4d65-a4d5-70967df67e1f, session_id: a9be0b57-4663-4b9a-b7eb-c2303d240fbe, gen: 1 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:04.49355752Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:48627): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.493591035Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:58719","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.521563816Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from eecd5e6a-956e-40b3-99d1-acc4c83ef30a with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.52201724Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: eecd5e6a-956e-40b3-99d1-acc4c83ef30a, session_id: 07c78e47-ca8f-4f63-8301-9c475d188349, gen: 3 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.522056394Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: eecd5e6a-956e-40b3-99d1-acc4c83ef30a, session_id: 07c78e47-ca8f-4f63-8301-9c475d188349, gen: 3 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.522477452Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1, 1, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.523202506Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.52323849Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: eecd5e6a-956e-40b3-99d1-acc4c83ef30a, session_id: 07c78e47-ca8f-4f63-8301-9c475d188349, gen: 3 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:13.663331518Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: eecd5e6a-956e-40b3-99d1-acc4c83ef30a, session_id: 07c78e47-ca8f-4f63-8301-9c475d188349, gen: 3 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:13.663401148Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:58719): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:13.663428834Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:37683","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.688757253Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 4109034b-6701-4b27-aba5-74837fe01eb4 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.689248721Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 4109034b-6701-4b27-aba5-74837fe01eb4, session_id: cc51dfda-de0b-4b8d-83ef-581ede764d24, gen: 5 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.689290085Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 4109034b-6701-4b27-aba5-74837fe01eb4, session_id: cc51dfda-de0b-4b8d-83ef-581ede764d24, gen: 5 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.689672079Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [11, 21, 31, 41, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.690486039Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:18.690519564Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 4109034b-6701-4b27-aba5-74837fe01eb4, session_id: cc51dfda-de0b-4b8d-83ef-581ede764d24, gen: 5 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:40.454138747Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 4109034b-6701-4b27-aba5-74837fe01eb4, session_id: cc51dfda-de0b-4b8d-83ef-581ede764d24, gen: 5 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:40.454206607Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:37683): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:40.45424999Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:35974","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.482729346Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 05da3c54-7295-4340-89e1-fb84f122181b with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.483216465Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 05da3c54-7295-4340-89e1-fb84f122181b, session_id: 187e9d97-0c3f-4c59-a0ca-b3ea507a3bea, gen: 7 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.48325078Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 05da3c54-7295-4340-89e1-fb84f122181b, session_id: 187e9d97-0c3f-4c59-a0ca-b3ea507a3bea, gen: 7 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.483680827Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [236, 236, 236, 236, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.48441021Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.484443445Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 05da3c54-7295-4340-89e1-fb84f122181b, session_id: 187e9d97-0c3f-4c59-a0ca-b3ea507a3bea, gen: 7 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:58.592689364Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 05da3c54-7295-4340-89e1-fb84f122181b, session_id: 187e9d97-0c3f-4c59-a0ca-b3ea507a3bea, gen: 7 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:58.592752635Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:35974): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.592779851Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:46211","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.622911985Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.623444057Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 9 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.623472303Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 9 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.623972229Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [692, 1050, 1307, 1463, 1518]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.624887395Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.62492395Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 9 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:03.651584138Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 9 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:03.651627682Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:46211): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:03.651652568Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:59455","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.652751992Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.653260467Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 10 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.653305401Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.653783941Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1521, 1523, 1523, 1523, 1521]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.654582244Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.654610699Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 10 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:04.6810562Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 10 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:04.68112013Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:59455): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:04.681152126Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:62600","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.682394908Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.682679726Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 11 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.682719311Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 11 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.683264531Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1526, 1526, 1524, 1526, 1526]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.68395Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.683986235Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 11 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:05.71025918Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 11 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:05.710303824Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:62600): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.71032924Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:35731","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.710904571Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.71111301Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 12 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.711151984Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 12 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.711561464Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1527, 1527, 1527, 1527, 1527]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.712087937Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.712119962Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 12 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:06.734254385Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 12 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:06.734306867Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:35731): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.734322895Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:40916","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.735419929Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.735653305Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 13 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.735680431Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 13 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.736170939Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1528, 1528, 1528, 1528, 1528]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.736734386Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.736767321Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 13 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:07.763879414Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 13 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:07.763945514Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:40916): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:07.76397089Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:36402","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.765592927Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 37aca9bf-c672-47ad-9284-10b2c9e487c0 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.765867587Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 14 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.765902582Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 14 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.766588252Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1530, 1530, 1530, 1530, 1530]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.767515425Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.76755267Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 14 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:12.801075522Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 37aca9bf-c672-47ad-9284-10b2c9e487c0, session_id: 18c861aa-82e8-476c-9bbd-046c24dc9066, gen: 14 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:12.80115661Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:36402): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:12.801187516Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:43012","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.871392198Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 258ec053-a81b-4b6d-9b49-5c8db2728492 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.872673211Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 4a0acb9a-13ec-4284-bccd-11654c0d105f, gen: 21 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.872715135Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 4a0acb9a-13ec-4284-bccd-11654c0d105f, gen: 21 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.873254665Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1533, 1533, 1532, 1532, 1532]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.873944054Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:17.873978899Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:62487","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.634568226Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 258ec053-a81b-4b6d-9b49-5c8db2728492 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.634906896Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.63495087Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 4a0acb9a-13ec-4284-bccd-11654c0d105f, gen: 21 } to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:18.635399784Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Signaling to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 4a0acb9a-13ec-4284-bccd-11654c0d105f, gen: 21 } thread that UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:18.635449197Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.635474203Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 4a0acb9a-13ec-4284-bccd-11654c0d105f, gen: 21 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:18.635500989Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:43012): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.635521486Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1535, 1535, 1535, 1535, 1535]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.636056138Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:18.636087723Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:54271","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.413200287Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 258ec053-a81b-4b6d-9b49-5c8db2728492 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.413516991Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.413565173Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:19.413893055Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Signaling to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 } thread that UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:19.413937709Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.413967305Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: ef5ac703-b116-435e-8464-fc6caedf4cd2, gen: 22 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:19.414005269Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:62487): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.414034344Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1538, 1538, 1538, 1538, 1537]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.414682249Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:19.414718504Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:47857","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.159530536Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 258ec053-a81b-4b6d-9b49-5c8db2728492 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.159697942Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.159741036Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.16005064Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Signaling to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 } thread that UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.160095644Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.160123209Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b1a67dd2-8b5b-4a54-82a1-51f1d5db18e7, gen: 23 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.160155315Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:54271): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.16018334Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1540, 1540, 1540, 1540, 1540]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.160643363Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.160674408Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:43886","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.914856636Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 258ec053-a81b-4b6d-9b49-5c8db2728492 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915018212Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915057056Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.915298871Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Signaling to UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 } thread that UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.915345554Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915378539Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: 21494608-fe44-47d0-883d-7400224048be, gen: 24 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:20.915412814Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:47857): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915446109Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1543, 1543, 1543, 1543, 1543]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915866427Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:20.915900522Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:21.6947562Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 258ec053-a81b-4b6d-9b49-5c8db2728492, session_id: b2dfdc0c-c20f-4881-953e-898cded4cd14, gen: 25 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:21.694830439Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:43886): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.694901719Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:33440","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.725903961Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 355080b1-61d0-4263-b20d-52e126bb045e with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.726321579Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 355080b1-61d0-4263-b20d-52e126bb045e, session_id: 1c33baf1-8692-4a65-a571-f6e65b0f10d7, gen: 31 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.726371632Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 355080b1-61d0-4263-b20d-52e126bb045e, session_id: 1c33baf1-8692-4a65-a571-f6e65b0f10d7, gen: 31 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.726813397Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1545, 1545, 1545, 1545, 1545]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.727489388Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.727516584Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 355080b1-61d0-4263-b20d-52e126bb045e, session_id: 1c33baf1-8692-4a65-a571-f6e65b0f10d7, gen: 31 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:25.74812137Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 355080b1-61d0-4263-b20d-52e126bb045e, session_id: 1c33baf1-8692-4a65-a571-f6e65b0f10d7, gen: 31 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:25.7481841Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:33440): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.748218235Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:34453","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.779422858Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.77988144Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9, session_id: 551a51a7-ba20-4ad9-867e-02c79a299d49, gen: 32 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.779912876Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9, session_id: 551a51a7-ba20-4ad9-867e-02c79a299d49, gen: 32 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.780276132Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1546, 1546, 1546, 1546, 1546]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.780909989Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:25.780940165Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"connection (127.0.0.1:34453) Exits with error: Error reading from Upstairs: Connection reset by peer (os error 131)","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:14:25.794619658Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"accepted connection from 127.0.0.1:42972","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.014643319Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"Connection request from 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25 with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.015237492Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.015277677Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9, session_id: 551a51a7-ba20-4ad9-867e-02c79a299d49, gen: 32 } to UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:26.015614247Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Signaling to UpstairsConnection { upstairs_id: 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9, session_id: 551a51a7-ba20-4ad9-867e-02c79a299d49, gen: 32 } thread that UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:26.015655681Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Error while signaling to UpstairsConnection { upstairs_id: 6f5fc00c-ee0e-49f0-b9b1-12a61ddc86b9, session_id: 551a51a7-ba20-4ad9-867e-02c79a299d49, gen: 32 } thread: UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 }","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:14:26.015682607Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.015698095Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Current flush_numbers [0..12]: [1546, 1546, 1546, 1546, 1546]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.016315134Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:26.016358598Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"proc"} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.022664972Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.076633334Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.079363214Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/004.copy\" to \"/var/tmp/test_up/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.079893346Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/004.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.081500551Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/004.replace\" to \"/var/tmp/test_up/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.08306911Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.087919239Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.120643618Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.12376787Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/001.copy\" to \"/var/tmp/test_up/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.124337706Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/001.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.126212401Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/001.replace\" to \"/var/tmp/test_up/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.127246529Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.131270979Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.163798927Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.166314528Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/003.copy\" to \"/var/tmp/test_up/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.166772081Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/003.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.168447255Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/003.replace\" to \"/var/tmp/test_up/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.170971714Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140} {"msg":"upstairs UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:32.1837483Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 67ea4f58-bfe5-4ba3-b3c0-0e4d244efd25, session_id: 1dcbb3d8-950c-4610-8723-dd7ab26ccbdc, gen: 33 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:32.183812651Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140,"task":"main"} {"msg":"connection (127.0.0.1:42972): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:32.183846236Z","hostname":"ip-10-150-1-30.us-west-2.compute.internal","pid":1140}