{"msg":"Opened existing region file \"/var/tmp/dsc/region/8840/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.006950952Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.007128995Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.007143263Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"UUID: 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.007954875Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Blocks per extent:100 Total Extents: 15","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.007971464Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\nCommit timestamp: 2024-02-28T21:51:33.000000000Z branch: main\nrustc: 1.75.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.008208001Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.008235138Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12840 for path:\"/var/tmp/dsc/region/8840\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.009226114Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.009806648Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12840","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.010182522Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.01020264Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8840","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:46.010216929Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"accepted connection from 127.0.0.1:40511","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.381328901Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"Connection request from 579965c0-1c5c-430d-a8b8-3247e0a345af with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.381784528Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.381816465Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.382319426Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.383265246Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.383320541Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.385016378Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.434841625Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.518381243Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/000.copy\" to \"/var/tmp/dsc/region/8840/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.519002583Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/000.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.524526644Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/000.replace\" to \"/var/tmp/dsc/region/8840/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.549884915Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.618614871Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.650702337Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.678531742Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/001.copy\" to \"/var/tmp/dsc/region/8840/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.679192808Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/001.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.681547443Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/001.replace\" to \"/var/tmp/dsc/region/8840/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.720849188Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.729848106Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.762409217Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.811732542Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/002.copy\" to \"/var/tmp/dsc/region/8840/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.812471351Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/002.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.842880878Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/002.replace\" to \"/var/tmp/dsc/region/8840/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.871811987Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.902308806Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.932451648Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.960596582Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/003.copy\" to \"/var/tmp/dsc/region/8840/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.961640452Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/003.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.964112375Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/003.replace\" to \"/var/tmp/dsc/region/8840/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:52.988328795Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.023538833Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.056908366Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.063118201Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/004.copy\" to \"/var/tmp/dsc/region/8840/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.063637961Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/004.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.065294833Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/004.replace\" to \"/var/tmp/dsc/region/8840/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.095552294Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.159582841Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.190895031Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.197706739Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/005.copy\" to \"/var/tmp/dsc/region/8840/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.198164345Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/005.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.224093041Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/005.replace\" to \"/var/tmp/dsc/region/8840/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.255903914Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.330394238Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.360450899Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.397030915Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/006.copy\" to \"/var/tmp/dsc/region/8840/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.3979116Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/006.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.401233322Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/006.replace\" to \"/var/tmp/dsc/region/8840/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.433102749Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.469027778Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.500556578Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.529952442Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/007.copy\" to \"/var/tmp/dsc/region/8840/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.53049719Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/007.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.53279499Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/007.replace\" to \"/var/tmp/dsc/region/8840/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.570363881Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1620 job Read for connection UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:53.574882698Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"role":"work"} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.577714827Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.609545947Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.64041975Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/008.copy\" to \"/var/tmp/dsc/region/8840/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.641499286Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/008.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.669086364Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/008.replace\" to \"/var/tmp/dsc/region/8840/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.691206165Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.75274534Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.78541943Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.814543Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/009.copy\" to \"/var/tmp/dsc/region/8840/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.815622607Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/009.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.818225528Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/009.replace\" to \"/var/tmp/dsc/region/8840/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.844569814Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.927944137Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.957703807Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.983185775Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00A.copy\" to \"/var/tmp/dsc/region/8840/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.983748102Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00A.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.986090417Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00A.replace\" to \"/var/tmp/dsc/region/8840/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.03397817Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.043767822Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.075175444Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.112565952Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00B.copy\" to \"/var/tmp/dsc/region/8840/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.113140777Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00B.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.116495336Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00B.replace\" to \"/var/tmp/dsc/region/8840/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.139425419Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.170302271Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.204441231Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.232798944Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00C.copy\" to \"/var/tmp/dsc/region/8840/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.233376769Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00C.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.236437636Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00C.replace\" to \"/var/tmp/dsc/region/8840/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.26687797Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.298534878Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.331007317Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.360425069Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00D.copy\" to \"/var/tmp/dsc/region/8840/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.361023841Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00D.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.363657999Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00D.replace\" to \"/var/tmp/dsc/region/8840/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.390294598Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.500942049Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.530977131Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.565448628Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00E.copy\" to \"/var/tmp/dsc/region/8840/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.565996676Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00E.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.569694302Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00E.replace\" to \"/var/tmp/dsc/region/8840/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:54.595316457Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"3386 job Read for connection UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:00.381066065Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"role":"work"} {"msg":"5559 job Write for connection UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:14.601653351Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"role":"work"} {"msg":"5560 job Flush for connection UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:14.601698597Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"role":"work"} {"msg":"connection (127.0.0.1:40511) Exits with error: framed write task ended: Ok(Err(Broken pipe (os error 32)))","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:13:23.75266382Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"accepted connection from 127.0.0.1:50347","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.8121587Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"Connection request from 579965c0-1c5c-430d-a8b8-3247e0a345af with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.812213955Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.812231253Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:30.812457012Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Signaling to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } thread that UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:30.81247959Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Error while signaling to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } thread: UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 }","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:13:30.812495488Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.812513637Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Current flush_numbers [0..12]: [675, 675, 675, 675, 674, 673, 670, 674, 675, 675, 670, 675]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.812978112Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.81299664Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.815803171Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.84681781Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.854072365Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/000.copy\" to \"/var/tmp/dsc/region/8840/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.854524332Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/000.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.856048026Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/000.replace\" to \"/var/tmp/dsc/region/8840/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.859223202Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.863984916Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.894222319Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.901842829Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/001.copy\" to \"/var/tmp/dsc/region/8840/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.90235944Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/001.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.904676838Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/001.replace\" to \"/var/tmp/dsc/region/8840/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.907118764Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.915472014Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.947755401Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.954692777Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/002.copy\" to \"/var/tmp/dsc/region/8840/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.955253153Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/002.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.956685456Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/002.replace\" to \"/var/tmp/dsc/region/8840/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.95977761Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.966925525Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:30.998541556Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.006516893Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/003.copy\" to \"/var/tmp/dsc/region/8840/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.00696493Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/003.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.008352597Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/003.replace\" to \"/var/tmp/dsc/region/8840/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.011563539Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.018916565Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.053083612Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.060340386Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/004.copy\" to \"/var/tmp/dsc/region/8840/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.061401935Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/004.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.063140048Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/004.replace\" to \"/var/tmp/dsc/region/8840/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.06583413Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.077404932Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.109061709Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.115363476Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/005.copy\" to \"/var/tmp/dsc/region/8840/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.115826661Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/005.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.11730095Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/005.replace\" to \"/var/tmp/dsc/region/8840/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.119902031Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.127995306Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.162827099Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.169862515Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/006.copy\" to \"/var/tmp/dsc/region/8840/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.170315562Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/006.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.171923998Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/006.replace\" to \"/var/tmp/dsc/region/8840/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.174465244Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.185276369Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.217370454Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.224153385Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/007.copy\" to \"/var/tmp/dsc/region/8840/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.224731059Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/007.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.226756385Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/007.replace\" to \"/var/tmp/dsc/region/8840/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.229395352Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.236470255Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.266544774Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.274179133Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/008.copy\" to \"/var/tmp/dsc/region/8840/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.274664546Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/008.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.276432777Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/008.replace\" to \"/var/tmp/dsc/region/8840/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.278437985Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.284837831Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.313903267Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.320358789Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/009.copy\" to \"/var/tmp/dsc/region/8840/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.320859051Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/009.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.32253682Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/009.replace\" to \"/var/tmp/dsc/region/8840/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.325077587Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.330932566Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.36167988Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.3689875Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00A.copy\" to \"/var/tmp/dsc/region/8840/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.369490572Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00A.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.37129329Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00A.replace\" to \"/var/tmp/dsc/region/8840/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.373633146Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.383652516Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.414560905Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.421003738Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00B.copy\" to \"/var/tmp/dsc/region/8840/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.421472563Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00B.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.423221635Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00B.replace\" to \"/var/tmp/dsc/region/8840/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.425943584Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.43298369Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.465094404Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.472253718Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00C.copy\" to \"/var/tmp/dsc/region/8840/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.472744351Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00C.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.474145497Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00C.replace\" to \"/var/tmp/dsc/region/8840/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.477056738Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.483188791Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.514317449Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.522128281Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00D.copy\" to \"/var/tmp/dsc/region/8840/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.522669819Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00D.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.524766148Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00D.replace\" to \"/var/tmp/dsc/region/8840/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.526887425Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.536880988Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.567444229Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.575990651Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00E.copy\" to \"/var/tmp/dsc/region/8840/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.576527639Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00E.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.578215208Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00E.replace\" to \"/var/tmp/dsc/region/8840/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:31.580662023Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.748425536Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:57927"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.748993641Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/0/files","method":"GET","req_id":"1af0025c-2fc0-44e4-b24d-1f3147e6dde9","remote_addr":"127.0.0.1:57927","local_addr":"0.0.0.0:12840","task":"repair","latency_us":302,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.749798084Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/0/data","method":"GET","req_id":"df5f25e1-a2f6-4407-96e9-27383819ae6b","remote_addr":"127.0.0.1:57927","local_addr":"0.0.0.0:12840","task":"repair","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.756183953Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/0/repair-ready","method":"GET","req_id":"bd72da1c-b1ff-4065-8ed9-ee64272af25b","remote_addr":"127.0.0.1:57927","local_addr":"0.0.0.0:12840","task":"repair","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.797532722Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:59425"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.797859251Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/1/files","method":"GET","req_id":"0f6a51ef-e76e-4360-8a6e-3f963025656f","remote_addr":"127.0.0.1:59425","local_addr":"0.0.0.0:12840","task":"repair","latency_us":156,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.798623637Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/1/data","method":"GET","req_id":"e326ce90-2c61-43ea-98c7-8886be974465","remote_addr":"127.0.0.1:59425","local_addr":"0.0.0.0:12840","task":"repair","latency_us":242,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.805758034Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/1/repair-ready","method":"GET","req_id":"e4189fa6-191a-4f1e-8f0f-e6fcef075c6d","remote_addr":"127.0.0.1:59425","local_addr":"0.0.0.0:12840","task":"repair","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.846176262Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:34607"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.846493762Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/2/files","method":"GET","req_id":"f6a1b6bd-aa4f-46a1-9588-7f0be95212fa","remote_addr":"127.0.0.1:34607","local_addr":"0.0.0.0:12840","task":"repair","latency_us":185,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.847208653Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/2/data","method":"GET","req_id":"0b38dedb-0dec-4c21-89f8-6c47cf2f3194","remote_addr":"127.0.0.1:34607","local_addr":"0.0.0.0:12840","task":"repair","latency_us":225,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.853913181Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/2/repair-ready","method":"GET","req_id":"7765e2a8-a468-4fe3-8f2a-734555b885fa","remote_addr":"127.0.0.1:34607","local_addr":"0.0.0.0:12840","task":"repair","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.894850889Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:46855"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.895229833Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/3/files","method":"GET","req_id":"2f6bc268-3c8b-4d52-8b54-d4d2f48e22ad","remote_addr":"127.0.0.1:46855","local_addr":"0.0.0.0:12840","task":"repair","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.895926986Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/3/data","method":"GET","req_id":"0bd0024b-285e-4fb8-92ef-1587c1633831","remote_addr":"127.0.0.1:46855","local_addr":"0.0.0.0:12840","task":"repair","latency_us":215,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.903101769Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/3/repair-ready","method":"GET","req_id":"41c775bf-af4c-45e4-8fde-b28c375050b3","remote_addr":"127.0.0.1:46855","local_addr":"0.0.0.0:12840","task":"repair","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.94589372Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:55688"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.94631398Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/4/files","method":"GET","req_id":"c4fde913-6859-4500-9398-9df00d213b41","remote_addr":"127.0.0.1:55688","local_addr":"0.0.0.0:12840","task":"repair","latency_us":173,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.94703912Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/4/data","method":"GET","req_id":"8bee8abc-a079-4e6f-840e-1a605f0ac181","remote_addr":"127.0.0.1:55688","local_addr":"0.0.0.0:12840","task":"repair","latency_us":206,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.952908498Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/4/repair-ready","method":"GET","req_id":"2518cf45-a20c-41b9-95f6-0c3e1adea35e","remote_addr":"127.0.0.1:55688","local_addr":"0.0.0.0:12840","task":"repair","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.995613657Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:56277"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.996046516Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/5/files","method":"GET","req_id":"60cdb82e-ff1d-44d5-8662-0ac7991511b8","remote_addr":"127.0.0.1:56277","local_addr":"0.0.0.0:12840","task":"repair","latency_us":177,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:37.996851039Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/5/data","method":"GET","req_id":"516d6cd7-bbd2-4beb-a6e5-370aa250a140","remote_addr":"127.0.0.1:56277","local_addr":"0.0.0.0:12840","task":"repair","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.003354566Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/5/repair-ready","method":"GET","req_id":"700ea383-8c3e-41a2-b570-52392deaecf0","remote_addr":"127.0.0.1:56277","local_addr":"0.0.0.0:12840","task":"repair","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.045625326Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:48648"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.045909499Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/6/files","method":"GET","req_id":"ad8da55f-e896-48c7-9ba9-061719525e3b","remote_addr":"127.0.0.1:48648","local_addr":"0.0.0.0:12840","task":"repair","latency_us":148,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.046612842Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/6/data","method":"GET","req_id":"b658a7d4-bfee-4d4d-b440-4e901e496538","remote_addr":"127.0.0.1:48648","local_addr":"0.0.0.0:12840","task":"repair","latency_us":218,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.052738165Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/6/repair-ready","method":"GET","req_id":"dded4b7a-74a6-4a16-8885-9e40c8525672","remote_addr":"127.0.0.1:48648","local_addr":"0.0.0.0:12840","task":"repair","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.093206688Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:60800"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.093635918Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/7/files","method":"GET","req_id":"154c27ae-1871-4f69-be06-c95419fc67ae","remote_addr":"127.0.0.1:60800","local_addr":"0.0.0.0:12840","task":"repair","latency_us":164,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.094389025Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/7/data","method":"GET","req_id":"302e1d03-3604-46fa-9a0f-4a67155ebe63","remote_addr":"127.0.0.1:60800","local_addr":"0.0.0.0:12840","task":"repair","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.100880853Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/7/repair-ready","method":"GET","req_id":"1588d772-b87f-40e5-bdeb-1b273f68049b","remote_addr":"127.0.0.1:60800","local_addr":"0.0.0.0:12840","task":"repair","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.143581183Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:36114"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.144085135Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/8/files","method":"GET","req_id":"1951ab19-bc0f-4f77-a6e9-9dce6c43807c","remote_addr":"127.0.0.1:36114","local_addr":"0.0.0.0:12840","task":"repair","latency_us":215,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.144945612Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/8/data","method":"GET","req_id":"34fe476b-39e0-49ff-9e65-105243591401","remote_addr":"127.0.0.1:36114","local_addr":"0.0.0.0:12840","task":"repair","latency_us":228,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.152090818Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/8/repair-ready","method":"GET","req_id":"56ec19ee-5a14-45b2-b74f-261de152810f","remote_addr":"127.0.0.1:36114","local_addr":"0.0.0.0:12840","task":"repair","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.196733942Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:64562"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.197197337Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/9/files","method":"GET","req_id":"d9b2eeaf-4cf9-4f34-a334-df5ed78107b8","remote_addr":"127.0.0.1:64562","local_addr":"0.0.0.0:12840","task":"repair","latency_us":180,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.198014109Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/9/data","method":"GET","req_id":"b49139f9-9421-49e9-b6c9-3f3d1ca929a1","remote_addr":"127.0.0.1:64562","local_addr":"0.0.0.0:12840","task":"repair","latency_us":208,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.20436844Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/9/repair-ready","method":"GET","req_id":"f05bb010-479b-46b7-a1db-83dcbd9878db","remote_addr":"127.0.0.1:64562","local_addr":"0.0.0.0:12840","task":"repair","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.254192817Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:52830"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.254666502Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/10/files","method":"GET","req_id":"fc62d178-01ba-4eeb-89aa-3dd9954d60c4","remote_addr":"127.0.0.1:52830","local_addr":"0.0.0.0:12840","task":"repair","latency_us":175,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.255471705Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/10/data","method":"GET","req_id":"a3821ccc-7a55-4444-b3c1-d4f673ebf34b","remote_addr":"127.0.0.1:52830","local_addr":"0.0.0.0:12840","task":"repair","latency_us":217,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.262344477Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/10/repair-ready","method":"GET","req_id":"498b9c06-9d3e-413f-a1e0-d0886f400a59","remote_addr":"127.0.0.1:52830","local_addr":"0.0.0.0:12840","task":"repair","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.305444538Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:48551"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.305706343Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/11/files","method":"GET","req_id":"b605aff8-c90a-40b4-9c78-2d8f8b069a55","remote_addr":"127.0.0.1:48551","local_addr":"0.0.0.0:12840","task":"repair","latency_us":132,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.306455141Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/11/data","method":"GET","req_id":"579397e1-718b-4c73-81bb-3ad38e68b930","remote_addr":"127.0.0.1:48551","local_addr":"0.0.0.0:12840","task":"repair","latency_us":223,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.312576914Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/11/repair-ready","method":"GET","req_id":"e3bf843e-f33c-4b3a-b1ff-097dd52ef045","remote_addr":"127.0.0.1:48551","local_addr":"0.0.0.0:12840","task":"repair","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.352684572Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:37572"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.353096223Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/12/files","method":"GET","req_id":"b1a87109-a3a5-4ec1-bd70-f3f0c46cb9bb","remote_addr":"127.0.0.1:37572","local_addr":"0.0.0.0:12840","task":"repair","latency_us":162,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.353935253Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/12/data","method":"GET","req_id":"869811e9-2362-4a11-a421-8478e1f3fab6","remote_addr":"127.0.0.1:37572","local_addr":"0.0.0.0:12840","task":"repair","latency_us":241,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.360185754Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/12/repair-ready","method":"GET","req_id":"3f0d409a-c716-48ad-a9f0-cab3e3f290df","remote_addr":"127.0.0.1:37572","local_addr":"0.0.0.0:12840","task":"repair","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.401937705Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:49821"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.402362074Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/13/files","method":"GET","req_id":"e189a522-3804-4d44-afaf-3ad636892473","remote_addr":"127.0.0.1:49821","local_addr":"0.0.0.0:12840","task":"repair","latency_us":177,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.403149038Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/13/data","method":"GET","req_id":"16367820-dc40-4b15-b514-550a8077b427","remote_addr":"127.0.0.1:49821","local_addr":"0.0.0.0:12840","task":"repair","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.409959126Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/13/repair-ready","method":"GET","req_id":"4a841537-5224-472b-beac-6a0bba16a27d","remote_addr":"127.0.0.1:49821","local_addr":"0.0.0.0:12840","task":"repair","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.452915011Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:54812"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.453361268Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/14/files","method":"GET","req_id":"4c6bb263-c864-44a4-abe2-9370cbe308e8","remote_addr":"127.0.0.1:54812","local_addr":"0.0.0.0:12840","task":"repair","latency_us":168,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.454189909Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/14/data","method":"GET","req_id":"89373602-7d10-4083-8870-2be2532d5067","remote_addr":"127.0.0.1:54812","local_addr":"0.0.0.0:12840","task":"repair","latency_us":252,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.461167621Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/14/repair-ready","method":"GET","req_id":"da68c42c-dead-43bb-af75-e9805a273612","remote_addr":"127.0.0.1:54812","local_addr":"0.0.0.0:12840","task":"repair","latency_us":75,"response_code":"200"} {"msg":"connection (127.0.0.1:50347) Exits with error: Error reading from Upstairs: Connection reset by peer (os error 131)","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:13:50.562845755Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"accepted connection from 127.0.0.1:37311","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.447257244Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"Connection request from 579965c0-1c5c-430d-a8b8-3247e0a345af with version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.447398021Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"upstairs UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } connected, version 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.447420298Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:58.44761205Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Signaling to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } thread that UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:58.447635568Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Error while signaling to UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } thread: UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 }","v":0,"name":"crucible","level":50,"time":"2024-02-28T22:13:58.447651356Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.447664485Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Current flush_numbers [0..12]: [1549, 1548, 1551, 1548, 1548, 1543, 1549, 1547, 1551, 1547, 1551, 1551]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.448404194Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Downstairs has completed Negotiation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.448431532Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"proc"} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.45043335Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.483792464Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.489543954Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/000.copy\" to \"/var/tmp/dsc/region/8840/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.490048465Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/000.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.492854136Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/000.replace\" to \"/var/tmp/dsc/region/8840/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.495043307Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.502749098Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.532918848Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.539665762Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/001.copy\" to \"/var/tmp/dsc/region/8840/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.540180493Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/001.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.542063752Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/001.replace\" to \"/var/tmp/dsc/region/8840/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.545075343Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.550262906Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.581010991Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.587453034Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/002.copy\" to \"/var/tmp/dsc/region/8840/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.587963345Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/002.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.589775541Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/002.replace\" to \"/var/tmp/dsc/region/8840/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.592107328Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.597738548Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.628526689Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.635900203Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/003.copy\" to \"/var/tmp/dsc/region/8840/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.636374907Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/003.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.637571082Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/003.replace\" to \"/var/tmp/dsc/region/8840/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.642306299Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.650659839Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.682184509Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.688569517Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/004.copy\" to \"/var/tmp/dsc/region/8840/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.689040652Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/004.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.690185853Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/004.replace\" to \"/var/tmp/dsc/region/8840/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.692690382Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.698744063Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.729655012Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.735997284Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/005.copy\" to \"/var/tmp/dsc/region/8840/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.736505425Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/005.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.737820219Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/005.replace\" to \"/var/tmp/dsc/region/8840/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.741331803Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.749598561Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.780380982Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.788098703Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/006.copy\" to \"/var/tmp/dsc/region/8840/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.788595826Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/006.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.790319661Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/006.replace\" to \"/var/tmp/dsc/region/8840/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.792858937Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.802825403Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.833728442Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.84001447Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/007.copy\" to \"/var/tmp/dsc/region/8840/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.840555118Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/007.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.842375154Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/007.replace\" to \"/var/tmp/dsc/region/8840/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.84491882Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.850112213Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.881927295Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.889479712Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/008.copy\" to \"/var/tmp/dsc/region/8840/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.889940958Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/008.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.891498588Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/008.replace\" to \"/var/tmp/dsc/region/8840/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.894154074Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.901624318Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.93250776Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.939567184Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/009.copy\" to \"/var/tmp/dsc/region/8840/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.940041718Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/009.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.94148853Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/009.replace\" to \"/var/tmp/dsc/region/8840/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.944316699Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.951547996Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.990164887Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.996652665Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00A.copy\" to \"/var/tmp/dsc/region/8840/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.997110321Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00A.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:58.998515597Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00A.replace\" to \"/var/tmp/dsc/region/8840/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.001356565Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.012253471Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.047646811Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.055297947Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00B.copy\" to \"/var/tmp/dsc/region/8840/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.055742735Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00B.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.057450241Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00B.replace\" to \"/var/tmp/dsc/region/8840/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.060375051Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.076124372Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.10653347Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.113463726Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00C.copy\" to \"/var/tmp/dsc/region/8840/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.11394163Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00C.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.115629018Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00C.replace\" to \"/var/tmp/dsc/region/8840/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.118298743Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.126375769Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.158055654Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.165163713Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00D.copy\" to \"/var/tmp/dsc/region/8840/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.165671325Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00D.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.166939683Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00D.replace\" to \"/var/tmp/dsc/region/8840/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.169918488Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Created copy dir \"/var/tmp/dsc/region/8840/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.17898079Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.212242073Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.218948241Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"1 repair files downloaded, move directory \"/var/tmp/dsc/region/8840/00/000/00E.copy\" to \"/var/tmp/dsc/region/8840/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.219419696Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Copy files from \"/var/tmp/dsc/region/8840/00/000/00E.replace\" in \"/var/tmp/dsc/region/8840/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.221027862Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"Move directory \"/var/tmp/dsc/region/8840/00/000/00E.replace\" to \"/var/tmp/dsc/region/8840/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:59.223079835Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.39341939Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:33626"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.393865908Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/0/files","method":"GET","req_id":"7256fcef-7731-4888-82c4-6a2d6dc962e6","remote_addr":"127.0.0.1:33626","local_addr":"0.0.0.0:12840","task":"repair","latency_us":170,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.394591018Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/0/data","method":"GET","req_id":"d8186cfe-ec7b-4b78-8685-b575d1306a2b","remote_addr":"127.0.0.1:33626","local_addr":"0.0.0.0:12840","task":"repair","latency_us":220,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.401576989Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/0/repair-ready","method":"GET","req_id":"a066a1a9-c178-4431-9c2b-62450aaa9639","remote_addr":"127.0.0.1:33626","local_addr":"0.0.0.0:12840","task":"repair","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.442941627Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:52788"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.4432234Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/1/files","method":"GET","req_id":"724cf46d-d025-46eb-be3b-558f4272f061","remote_addr":"127.0.0.1:52788","local_addr":"0.0.0.0:12840","task":"repair","latency_us":159,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.443926142Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/1/data","method":"GET","req_id":"f954d60a-362d-4223-829e-3d05a6b8cea2","remote_addr":"127.0.0.1:52788","local_addr":"0.0.0.0:12840","task":"repair","latency_us":202,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.450082882Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/1/repair-ready","method":"GET","req_id":"b109b7ad-a342-4840-a953-aab557122730","remote_addr":"127.0.0.1:52788","local_addr":"0.0.0.0:12840","task":"repair","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.4917577Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:36559"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.492195948Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/2/files","method":"GET","req_id":"b3ef5f79-9409-4dde-948c-e155a04a42ba","remote_addr":"127.0.0.1:36559","local_addr":"0.0.0.0:12840","task":"repair","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.492932918Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/2/data","method":"GET","req_id":"69894ecf-2317-4c65-9b69-cfe2a09e6feb","remote_addr":"127.0.0.1:36559","local_addr":"0.0.0.0:12840","task":"repair","latency_us":210,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.499948735Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/2/repair-ready","method":"GET","req_id":"cb894174-566d-4d05-8a5d-0fd39885681e","remote_addr":"127.0.0.1:36559","local_addr":"0.0.0.0:12840","task":"repair","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.548050287Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:42035"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.548479346Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/3/files","method":"GET","req_id":"ebb7f5da-8dc4-432c-8e9d-f6d1401c304b","remote_addr":"127.0.0.1:42035","local_addr":"0.0.0.0:12840","task":"repair","latency_us":171,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.54926843Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/3/data","method":"GET","req_id":"1858551e-73b1-484f-a2b2-02783b0a6d04","remote_addr":"127.0.0.1:42035","local_addr":"0.0.0.0:12840","task":"repair","latency_us":249,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.555472286Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/3/repair-ready","method":"GET","req_id":"4f8f9a5d-3669-4d2a-afa7-90c74a1ac74a","remote_addr":"127.0.0.1:42035","local_addr":"0.0.0.0:12840","task":"repair","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.59864989Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:48762"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.59906351Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/4/files","method":"GET","req_id":"4719bab2-3ef2-42de-aed9-27b9305967bf","remote_addr":"127.0.0.1:48762","local_addr":"0.0.0.0:12840","task":"repair","latency_us":151,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.599775452Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/4/data","method":"GET","req_id":"ab310cd4-1a75-4774-8ccc-2b3d9a2cc1e6","remote_addr":"127.0.0.1:48762","local_addr":"0.0.0.0:12840","task":"repair","latency_us":232,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.607130467Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/4/repair-ready","method":"GET","req_id":"c64150a3-4309-4e64-94d9-48fee07e5fd6","remote_addr":"127.0.0.1:48762","local_addr":"0.0.0.0:12840","task":"repair","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.64718823Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:45669"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.647472373Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/5/files","method":"GET","req_id":"9bd28746-1294-48d7-8436-d5d9488dba56","remote_addr":"127.0.0.1:45669","local_addr":"0.0.0.0:12840","task":"repair","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.648118501Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/5/data","method":"GET","req_id":"528f2e54-c757-4bec-b13d-2f47be31e43b","remote_addr":"127.0.0.1:45669","local_addr":"0.0.0.0:12840","task":"repair","latency_us":195,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.654513019Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/5/repair-ready","method":"GET","req_id":"5241770c-3314-431a-945b-40f1e69f4242","remote_addr":"127.0.0.1:45669","local_addr":"0.0.0.0:12840","task":"repair","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.695497313Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:46753"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.695862707Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/6/files","method":"GET","req_id":"c8489044-f3e3-4285-8874-b489493738a5","remote_addr":"127.0.0.1:46753","local_addr":"0.0.0.0:12840","task":"repair","latency_us":132,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.696606846Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/6/data","method":"GET","req_id":"fd738028-7359-492e-b4c3-552e4599ced3","remote_addr":"127.0.0.1:46753","local_addr":"0.0.0.0:12840","task":"repair","latency_us":202,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.703168558Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/6/repair-ready","method":"GET","req_id":"6d494f71-21a7-4b00-9797-10a3681b4dcb","remote_addr":"127.0.0.1:46753","local_addr":"0.0.0.0:12840","task":"repair","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.743608374Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:65409"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.743991537Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/7/files","method":"GET","req_id":"8b7c1f1c-1b9b-4c2e-b5c4-c76df60dc4d2","remote_addr":"127.0.0.1:65409","local_addr":"0.0.0.0:12840","task":"repair","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.744699369Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/7/data","method":"GET","req_id":"797e6012-131e-40f1-8411-f40a5867bf3c","remote_addr":"127.0.0.1:65409","local_addr":"0.0.0.0:12840","task":"repair","latency_us":217,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.751404887Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/7/repair-ready","method":"GET","req_id":"5e843a31-a144-4f4b-8531-53eb34229259","remote_addr":"127.0.0.1:65409","local_addr":"0.0.0.0:12840","task":"repair","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.796933626Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:34044"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.797240546Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/8/files","method":"GET","req_id":"6e19d33d-08bc-423b-aa63-8f5ac1b176c9","remote_addr":"127.0.0.1:34044","local_addr":"0.0.0.0:12840","task":"repair","latency_us":171,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.797949968Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/8/data","method":"GET","req_id":"3130c0d4-4091-45e2-84ea-2584835b3a7f","remote_addr":"127.0.0.1:34044","local_addr":"0.0.0.0:12840","task":"repair","latency_us":214,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.804752527Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/8/repair-ready","method":"GET","req_id":"ae72298f-2a96-4b81-be22-6e1cce491333","remote_addr":"127.0.0.1:34044","local_addr":"0.0.0.0:12840","task":"repair","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.845756019Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:49296"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.846201826Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/9/files","method":"GET","req_id":"0ed90bfb-6cd5-4e22-bf74-fedbfe5e76de","remote_addr":"127.0.0.1:49296","local_addr":"0.0.0.0:12840","task":"repair","latency_us":182,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.846960264Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/9/data","method":"GET","req_id":"06871422-c97c-4710-a8c6-7cdb41dbd34d","remote_addr":"127.0.0.1:49296","local_addr":"0.0.0.0:12840","task":"repair","latency_us":213,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.854290591Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/9/repair-ready","method":"GET","req_id":"7eee7c53-5381-49db-9287-6f1e9f9de1a1","remote_addr":"127.0.0.1:49296","local_addr":"0.0.0.0:12840","task":"repair","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.895879507Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:58401"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.896136283Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/10/files","method":"GET","req_id":"9e872296-ef39-4225-b299-f1387a332ac4","remote_addr":"127.0.0.1:58401","local_addr":"0.0.0.0:12840","task":"repair","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.896801109Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/10/data","method":"GET","req_id":"c83d067d-0736-41fc-b37a-29009513204f","remote_addr":"127.0.0.1:58401","local_addr":"0.0.0.0:12840","task":"repair","latency_us":200,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.903092787Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/10/repair-ready","method":"GET","req_id":"de3d6148-7486-4cf0-9777-501dec5279e8","remote_addr":"127.0.0.1:58401","local_addr":"0.0.0.0:12840","task":"repair","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.947930021Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:43360"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.948348011Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/11/files","method":"GET","req_id":"27534cf3-db9c-4295-b4a9-3f9e1b650932","remote_addr":"127.0.0.1:43360","local_addr":"0.0.0.0:12840","task":"repair","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.949027876Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/11/data","method":"GET","req_id":"2db4aa84-88b3-464c-b255-5a3ce3e572e5","remote_addr":"127.0.0.1:43360","local_addr":"0.0.0.0:12840","task":"repair","latency_us":203,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.955051109Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/11/repair-ready","method":"GET","req_id":"6f3e2158-80df-48d3-907f-5a5903062821","remote_addr":"127.0.0.1:43360","local_addr":"0.0.0.0:12840","task":"repair","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.998492388Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:48249"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.998945665Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/12/files","method":"GET","req_id":"4abbc4d0-dac2-4311-9cc8-1f2503b6c148","remote_addr":"127.0.0.1:48249","local_addr":"0.0.0.0:12840","task":"repair","latency_us":175,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:05.999741798Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/12/data","method":"GET","req_id":"154acf3e-52b4-4908-b81f-81de8635afa9","remote_addr":"127.0.0.1:48249","local_addr":"0.0.0.0:12840","task":"repair","latency_us":211,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.007192215Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/12/repair-ready","method":"GET","req_id":"62282501-be05-472f-949b-8f0b45008677","remote_addr":"127.0.0.1:48249","local_addr":"0.0.0.0:12840","task":"repair","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.051308729Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:57839"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.051698851Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/13/files","method":"GET","req_id":"7224254d-ab53-46f1-8671-872fc0e04b87","remote_addr":"127.0.0.1:57839","local_addr":"0.0.0.0:12840","task":"repair","latency_us":147,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.052476767Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/13/data","method":"GET","req_id":"e3814489-0df5-4198-b132-1cf9a335c3ce","remote_addr":"127.0.0.1:57839","local_addr":"0.0.0.0:12840","task":"repair","latency_us":234,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.059265336Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/13/repair-ready","method":"GET","req_id":"ef751e04-8854-44e4-acbd-201a7141f597","remote_addr":"127.0.0.1:57839","local_addr":"0.0.0.0:12840","task":"repair","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.105235133Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"local_addr":"0.0.0.0:12840","task":"repair","remote_addr":"127.0.0.1:45392"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.105635245Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/14/files","method":"GET","req_id":"24d7dfdd-1d69-4ca9-99e4-18407273f3f8","remote_addr":"127.0.0.1:45392","local_addr":"0.0.0.0:12840","task":"repair","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.106401661Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/newextent/14/data","method":"GET","req_id":"84a3ba51-cff2-4a8b-a8db-f84d9014ec2f","remote_addr":"127.0.0.1:45392","local_addr":"0.0.0.0:12840","task":"repair","latency_us":216,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:06.112443882Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"uri":"/extent/14/repair-ready","method":"GET","req_id":"75a0c5c9-a682-4549-a6f5-8b129414ddb3","remote_addr":"127.0.0.1:45392","local_addr":"0.0.0.0:12840","task":"repair","latency_us":86,"response_code":"200"} {"msg":"upstairs UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } disconnected, 0 jobs left","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:11.139906147Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"upstairs UpstairsConnection { upstairs_id: 579965c0-1c5c-430d-a8b8-3247e0a345af, session_id: 0759252c-1f53-4fc2-b3d6-4192deae14e5, gen: 1 } was previously active, clearing","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:11.139966631Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138,"task":"main"} {"msg":"connection (127.0.0.1:37311): all done","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:11.140003727Z","hostname":"ip-10-150-1-81.us-west-2.compute.internal","pid":1138}