{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.96388321Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Opened existing region file \"/var/tmp/test_up/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.964352626Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.964391159Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.964601732Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.969416803Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Blocks per extent:10 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.969443886Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.97134477Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.971896915Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Using address: 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.971937612Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.980503436Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.982433659Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.982855271Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.982903579Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.983614932Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"accepted connection from 127.0.0.1:53963","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.137495187Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from ce75bfe1-267b-47f2-a887-b01a806a2fd5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.139245978Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: ce75bfe1-267b-47f2-a887-b01a806a2fd5, session_id: 45b16846-311d-4a4b-b611-d4e4c53f1050, gen: 1 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.139301493Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: ce75bfe1-267b-47f2-a887-b01a806a2fd5, session_id: 45b16846-311d-4a4b-b611-d4e4c53f1050, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.142503682Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.144375588Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.144722181Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: ce75bfe1-267b-47f2-a887-b01a806a2fd5, session_id: 45b16846-311d-4a4b-b611-d4e4c53f1050, gen: 1 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:16.240077887Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: ce75bfe1-267b-47f2-a887-b01a806a2fd5, session_id: 45b16846-311d-4a4b-b611-d4e4c53f1050, gen: 1 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:16.240129811Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:53963): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.240510724Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:49285","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.274891852Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 089d5dd4-e6c8-43ed-bb6a-15445207a6a9 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.276257931Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 089d5dd4-e6c8-43ed-bb6a-15445207a6a9, session_id: f329e082-1b6a-47a0-a358-018d30be0362, gen: 2 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.276313738Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 089d5dd4-e6c8-43ed-bb6a-15445207a6a9, session_id: f329e082-1b6a-47a0-a358-018d30be0362, gen: 2 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.276560755Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1, 1, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.277285616Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:16.277599658Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 089d5dd4-e6c8-43ed-bb6a-15445207a6a9, session_id: f329e082-1b6a-47a0-a358-018d30be0362, gen: 2 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:20.859614564Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 089d5dd4-e6c8-43ed-bb6a-15445207a6a9, session_id: f329e082-1b6a-47a0-a358-018d30be0362, gen: 2 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:20.859703719Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:49285): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.859746648Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:34440","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.892020782Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from be207a11-6438-4c72-9500-6e71fc38f10e with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.893167943Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: be207a11-6438-4c72-9500-6e71fc38f10e, session_id: f0963fe8-58df-44a2-b7af-f5b78b98f258, gen: 3 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.893222132Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: be207a11-6438-4c72-9500-6e71fc38f10e, session_id: f0963fe8-58df-44a2-b7af-f5b78b98f258, gen: 3 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.893907482Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [11, 21, 31, 41, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.894725303Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.894945504Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: be207a11-6438-4c72-9500-6e71fc38f10e, session_id: f0963fe8-58df-44a2-b7af-f5b78b98f258, gen: 3 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:02.503737593Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: be207a11-6438-4c72-9500-6e71fc38f10e, session_id: f0963fe8-58df-44a2-b7af-f5b78b98f258, gen: 3 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:02.503825522Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:34440): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.503936231Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:49013","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.544658847Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from a2c7e9d9-ef3a-40c2-bed2-cc2bc7d85f07 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.545801092Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: a2c7e9d9-ef3a-40c2-bed2-cc2bc7d85f07, session_id: fdb46626-6e31-4cec-80f8-2c81b8929e92, gen: 4 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.545845383Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: a2c7e9d9-ef3a-40c2-bed2-cc2bc7d85f07, session_id: fdb46626-6e31-4cec-80f8-2c81b8929e92, gen: 4 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.5463228Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [251, 251, 251, 251, 251]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.547040652Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:02.547299762Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: a2c7e9d9-ef3a-40c2-bed2-cc2bc7d85f07, session_id: fdb46626-6e31-4cec-80f8-2c81b8929e92, gen: 4 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.84592328Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: a2c7e9d9-ef3a-40c2-bed2-cc2bc7d85f07, session_id: fdb46626-6e31-4cec-80f8-2c81b8929e92, gen: 4 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.84599034Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:49013): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.846088176Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:47640","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.883268562Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.884240633Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 5 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.884299992Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 5 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.884915276Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [718, 1081, 1338, 1496, 1551]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.885792098Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.885997557Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 5 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.958549434Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 5 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.958614314Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:47640): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.958643397Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:43248","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.959920961Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.960142406Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 6 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.960183583Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 6 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.960712902Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1555, 1555, 1553, 1553, 1555]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.961435887Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.961699283Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 6 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:34.033426126Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 6 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:34.033495629Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:43248): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:34.033535215Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:60633","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.034557082Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.03473147Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 7 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.034777843Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 7 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.035436293Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1556, 1556, 1558, 1558, 1556]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.036237929Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:35.036508045Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 7 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:45.114777042Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 7 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:45.11482289Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:60633): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:45.114976468Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:47425","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.11644074Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.116652954Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 8 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.116694935Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 8 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.117458807Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1560, 1559, 1560, 1560, 1560]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.118200909Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.118383378Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 8 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:46.19126406Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 8 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:46.191325789Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:47425): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:46.191369748Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:36252","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.192767133Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.192985037Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 9 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.193039452Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 9 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.19364197Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1560, 1562, 1562, 1562, 1562]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.19441331Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.194643075Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 9 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:47.253773932Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 9 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:47.253816187Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:36252): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:47.253838315Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:42912","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.254967389Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 7a2c34a1-4b3a-4085-8ca1-2b015299928f with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255636617Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 10 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255694872Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255964779Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1563, 1563, 1563, 1563, 1563]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256753469Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256949303Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 10 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:52.35526876Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 7a2c34a1-4b3a-4085-8ca1-2b015299928f, session_id: f3e02821-73e5-4f08-90b0-aa3ad6512446, gen: 10 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:52.355355454Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:42912): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.355460816Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:60482","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.450797899Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"accepted connection from 127.0.0.1:33107","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.451429121Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"accepted connection from 127.0.0.1:63290","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.452003285Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.453500202Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.453560207Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:49114","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.454049685Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.454216241Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.454271453Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.454900771Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.454967464Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.455279615Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.455337553Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:36369","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.456267352Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.456768688Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.456822662Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.457272507Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1565, 1565, 1565, 1565, 1565]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.458409092Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:52.458750705Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:54.98323232Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Signaling to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } thread that UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:54.983294879Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:54.983334839Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:54.983380103Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:33107): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:54.983421908Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1570, 1571, 1570, 1570, 1570]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:54.984386752Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:54.984694405Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:47367","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:55.985500372Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:55.98559028Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:55.985636461Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:57.56548753Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Signaling to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } thread that UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:57.565556206Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.565604406Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:57.565653422Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:60482): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.565697205Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1576, 1577, 1577, 1577, 1576]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.566298162Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.566606403Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:44627","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:58.570251446Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:58.570421194Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:58.570486384Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:00.082288657Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Signaling to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } thread that UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:00.082346922Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:00.082386693Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:00.082429733Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:63290): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:00.082464808Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1583, 1583, 1582, 1583, 1583]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:00.083243839Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:00.083539081Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:41647","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:01.087035576Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:01.087446259Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:01.087505503Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:02.618651729Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Signaling to UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } thread that UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:02.618759086Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:02.618810526Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Another upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:02.618858997Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:49114): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:02.618894952Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1589, 1589, 1589, 1589, 1589]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:02.619541236Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:02.619761994Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"accepted connection from 127.0.0.1:35948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.624036444Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 10e06cb8-04a4-4806-a97a-3fbdd40729d5 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.624144937Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.624177488Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } cannot grab lock, does not match UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 }!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.10656843Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: e65cc2f2-f262-4266-b51d-152ba9bc4350, gen: 16 } disconnected, Upstairs is not active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107217248Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"connection (127.0.0.1:47367): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107461359Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.107644841Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: bd96b7f3-8b82-44b6-9280-e92a47f98c80, gen: 20 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.107692491Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:36369): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107734685Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } cannot grab work lock, 10e06cb8-04a4-4806-a97a-3fbdd40729d5 is not active!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.107775964Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 7e05ede5-4eef-44d8-a700-6ce785b04767, gen: 17 } disconnected, Upstairs is not active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107813685Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"connection (127.0.0.1:44627): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107848841Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } cannot grab work lock, 10e06cb8-04a4-4806-a97a-3fbdd40729d5 is not active!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.107885224Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 65e00098-b636-45df-9633-f212841e80a7, gen: 18 } disconnected, Upstairs is not active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.107920712Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"connection (127.0.0.1:41647): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.108250602Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } cannot grab work lock, 10e06cb8-04a4-4806-a97a-3fbdd40729d5 is not active!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:05.108307891Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 10e06cb8-04a4-4806-a97a-3fbdd40729d5, session_id: 857c23bd-bdb3-412f-b66d-7e906588cc47, gen: 19 } disconnected, Upstairs is not active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.108347418Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"connection (127.0.0.1:35948): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.108373636Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:45903","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.149222898Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 48015b0a-cfb5-441c-a91b-7d360cbb537b with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.150215704Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 48015b0a-cfb5-441c-a91b-7d360cbb537b, session_id: 5d670231-9bb9-4165-9bc4-24f095d9e952, gen: 26 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.150254549Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 48015b0a-cfb5-441c-a91b-7d360cbb537b, session_id: 5d670231-9bb9-4165-9bc4-24f095d9e952, gen: 26 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.150865977Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1595, 1595, 1595, 1595, 1595]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.151796316Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:05.152102266Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 48015b0a-cfb5-441c-a91b-7d360cbb537b, session_id: 5d670231-9bb9-4165-9bc4-24f095d9e952, gen: 26 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:09.206100088Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 48015b0a-cfb5-441c-a91b-7d360cbb537b, session_id: 5d670231-9bb9-4165-9bc4-24f095d9e952, gen: 26 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:09.206164967Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:45903): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.206206429Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:57903","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.277934334Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from a8713597-e5a4-42a1-a6ca-aa1496a26ddb with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.279088003Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: a8713597-e5a4-42a1-a6ca-aa1496a26ddb, session_id: aee7ea26-b49c-4d9f-9b0d-7b655dbe5ca4, gen: 27 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.27913095Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: a8713597-e5a4-42a1-a6ca-aa1496a26ddb, session_id: aee7ea26-b49c-4d9f-9b0d-7b655dbe5ca4, gen: 27 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.279962449Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1596, 1596, 1596, 1596, 1596]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.28083896Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.281166451Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"connection (127.0.0.1:57903) Exits with error: do_work_task task has ended: Ok(Err(Broken pipe (os error 32)))","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:26:09.321581034Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"accepted connection from 127.0.0.1:59919","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.768612111Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"Connection request from 461dba6d-98ee-4cf2-a187-c79aeb4bf70a with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.769763757Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.769814907Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: a8713597-e5a4-42a1-a6ca-aa1496a26ddb, session_id: aee7ea26-b49c-4d9f-9b0d-7b655dbe5ca4, gen: 27 } to UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:09.770256867Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Signaling to UpstairsConnection { upstairs_id: a8713597-e5a4-42a1-a6ca-aa1496a26ddb, session_id: aee7ea26-b49c-4d9f-9b0d-7b655dbe5ca4, gen: 27 } thread that UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:09.770311207Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Error while signaling to UpstairsConnection { upstairs_id: a8713597-e5a4-42a1-a6ca-aa1496a26ddb, session_id: aee7ea26-b49c-4d9f-9b0d-7b655dbe5ca4, gen: 27 } thread: SendError { .. }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:26:09.770342351Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } to active, discarding 2 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:09.770377495Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.77040521Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Current flush_numbers [0..12]: [1596, 1597, 1597, 1596, 1596]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.77129498Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:09.771498865Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"proc"} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.778684902Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"eid:0 Found repair files: [\"000\", \"000.db\"]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.88874Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/000.copy\" to \"/var/tmp/test_up/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.89533549Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/000.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.897496279Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/000.replace\" to \"/var/tmp/test_up/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.900144922Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.909700814Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"eid:4 Found repair files: [\"004\", \"004.db\"]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.977515492Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/004.copy\" to \"/var/tmp/test_up/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.983226427Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/004.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.985128781Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/004.replace\" to \"/var/tmp/test_up/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.987499346Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.997140474Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"eid:3 Found repair files: [\"003\", \"003.db\"]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.06180446Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/003.copy\" to \"/var/tmp/test_up/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.068179595Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/003.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.071591273Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/003.replace\" to \"/var/tmp/test_up/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.074248135Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.084165962Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"eid:2 Found repair files: [\"002\", \"002.db\"]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.148864802Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/002.copy\" to \"/var/tmp/test_up/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.154644431Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/002.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.157018831Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/002.replace\" to \"/var/tmp/test_up/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.159346691Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Created copy dir \"/var/tmp/test_up/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.168549747Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"eid:1 Found repair files: [\"001\", \"001.db\"]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.234068895Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Repair files downloaded, move directory \"/var/tmp/test_up/8820/00/000/001.copy\" to \"/var/tmp/test_up/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.239484978Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Copy files from \"/var/tmp/test_up/8820/00/000/001.replace\" in \"/var/tmp/test_up/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.24135674Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"Move directory \"/var/tmp/test_up/8820/00/000/001.replace\" to \"/var/tmp/test_up/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.243907962Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162} {"msg":"upstairs UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:00.606746805Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 461dba6d-98ee-4cf2-a187-c79aeb4bf70a, session_id: 64614354-774d-4f1d-b460-8dd12e094501, gen: 28 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:00.606842636Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162,"task":"main"} {"msg":"connection (127.0.0.1:59919): all done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.60688067Z","hostname":"ip-10-150-1-13.us-west-2.compute.internal","pid":1162}