{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.834058498Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.834738325Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.834765372Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.838617248Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.838643665Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: dd1cf1330bf5c5b1551a0ad961a7aa7c6207feae\nCommit timestamp: 2025-01-23T18:31:41.000000000Z branch: mkeeter/early-io-rejection\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.88308344Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.883141784Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.884373122Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.884638543Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.884766649Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.884793247Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.88484968Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"accepted connection from 127.0.0.1:39521","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.884912494Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:39521): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.885150948Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"0"} {"msg":"Connection request from 054113b2-233d-4cd4-b6bc-cd6b68368ac1 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.885377054Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 054113b2-233d-4cd4-b6bc-cd6b68368ac1, session_id: 5d4119e3-33ed-49c9-bf5c-c22909abf080, gen: 193 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.88541196Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: 054113b2-233d-4cd4-b6bc-cd6b68368ac1, session_id: 5d4119e3-33ed-49c9-bf5c-c22909abf080, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.885869261Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [308, 317, 315, 317, 315, 320, 314, 310, 319, 299, 312, 319]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.886636158Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.887992923Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.889483632Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.891423084Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/003.copy\" to \"/var/tmp/test_repair/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.891953497Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/003.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.89201508Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/003.replace\" to \"/var/tmp/test_repair/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.892141297Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.893546076Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.894285137Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.896027509Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.896548753Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.89658014Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.896707906Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.89807476Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.898599763Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.900226448Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00C.copy\" to \"/var/tmp/test_repair/8820/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.900795007Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.900830853Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00C.replace\" to \"/var/tmp/test_repair/8820/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.900945271Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.902331202Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.902834278Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.90431069Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.904819275Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.904896867Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.905024073Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/018.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.906653118Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:24 Found repair files: [\"018\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.907239855Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 24 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.908690839Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/018.copy\" to \"/var/tmp/test_repair/8820/00/000/018.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.909293394Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/018.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.909322031Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/018.replace\" to \"/var/tmp/test_repair/8820/00/000/018.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.909447298Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.910705272Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.911398928Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.912857461Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.913464006Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.913489263Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.913651686Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.915013689Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.915511326Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.916883299Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00D.copy\" to \"/var/tmp/test_repair/8820/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.917358777Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.917389004Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00D.replace\" to \"/var/tmp/test_repair/8820/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.917493893Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.919176732Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.919640862Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.921425171Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.924269075Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.924298872Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.924428078Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.925738167Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.926478208Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.928329799Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.928859062Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.928902787Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.929017015Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.930481638Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.931037158Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.932612199Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.933273487Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.933315233Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:37.933485115Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446610533Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446661927Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446684355Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446701633Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446720281Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 12 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446739099Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446759527Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446780314Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446799503Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 24 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:38.446811261Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:41.98782624Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:41.987894393Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:41.98792511Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 054113b2-233d-4cd4-b6bc-cd6b68368ac1, session_id: 5d4119e3-33ed-49c9-bf5c-c22909abf080, gen: 193 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:41.987954307Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 054113b2-233d-4cd4-b6bc-cd6b68368ac1, session_id: 5d4119e3-33ed-49c9-bf5c-c22909abf080, gen: 193 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:41.987982224Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection from 127.0.0.1:49930","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.095207743Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:49930): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.095301933Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"1"} {"msg":"Connection request from 21ba841a-f90b-43ba-9fae-5e137049b152 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.095782601Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 21ba841a-f90b-43ba-9fae-5e137049b152, session_id: ec497450-cff2-43e5-b0b5-5325b1022a9c, gen: 194 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.095817817Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: 21ba841a-f90b-43ba-9fae-5e137049b152, session_id: ec497450-cff2-43e5-b0b5-5325b1022a9c, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.096221084Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [321, 317, 315, 321, 315, 320, 314, 321, 321, 299, 312, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.097144435Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.118477983Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.118515269Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.118538446Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 21ba841a-f90b-43ba-9fae-5e137049b152, session_id: ec497450-cff2-43e5-b0b5-5325b1022a9c, gen: 194 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.118562803Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 21ba841a-f90b-43ba-9fae-5e137049b152, session_id: ec497450-cff2-43e5-b0b5-5325b1022a9c, gen: 194 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.118588221Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection from 127.0.0.1:52613","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.223050997Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:52613): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.223143157Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"2"} {"msg":"Connection request from ae89cbb4-b949-4203-9df7-9ca21fd2bd1e with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.2235831Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: ae89cbb4-b949-4203-9df7-9ca21fd2bd1e, session_id: 7884102e-5996-4c22-a9e3-ab2fcf39c0aa, gen: 195 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.223611937Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: ae89cbb4-b949-4203-9df7-9ca21fd2bd1e, session_id: 7884102e-5996-4c22-a9e3-ab2fcf39c0aa, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.223995906Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [321, 317, 322, 322, 322, 322, 314, 321, 321, 299, 312, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.224799609Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.226175881Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.226773907Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.228431129Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/009.copy\" to \"/var/tmp/test_repair/8820/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.228972401Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/009.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.229019416Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/009.replace\" to \"/var/tmp/test_repair/8820/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.22916501Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.231521847Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.236196955Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.237863405Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.238364642Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.238399618Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.238537273Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/018.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.23987205Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:24 Found repair files: [\"018\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.240380535Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 24 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.242131347Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/018.copy\" to \"/var/tmp/test_repair/8820/00/000/018.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.242601916Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/018.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.242644052Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/018.replace\" to \"/var/tmp/test_repair/8820/00/000/018.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.242783877Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.244113144Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.244609131Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.246160744Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.246678208Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.246707805Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.246873847Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.248189556Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.248672234Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.250451183Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/019.copy\" to \"/var/tmp/test_repair/8820/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.250957538Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/019.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.250981926Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/019.replace\" to \"/var/tmp/test_repair/8820/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:42.25112804Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.766156494Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.766210598Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 9 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.766236115Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 24 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.766260963Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:42.766285Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.303700135Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:46.303762948Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.303821702Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: ae89cbb4-b949-4203-9df7-9ca21fd2bd1e, session_id: 7884102e-5996-4c22-a9e3-ab2fcf39c0aa, gen: 195 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.30384503Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: ae89cbb4-b949-4203-9df7-9ca21fd2bd1e, session_id: 7884102e-5996-4c22-a9e3-ab2fcf39c0aa, gen: 195 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.303871816Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.442893819Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:33545"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.443871925Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"uri":"/region-info","method":"GET","req_id":"01bf6683-d14d-45ea-ae27-7855c96947bd","remote_addr":"127.0.0.1:33545","local_addr":"0.0.0.0:12820","task":"repair","latency_us":258,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:54399","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.447237113Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:54399): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.447348261Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"3"} {"msg":"Connection request from e0e277c1-557f-4785-b433-9086b41beda9 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.447792923Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: e0e277c1-557f-4785-b433-9086b41beda9, session_id: 7ddaf088-1563-42cd-bdb8-f91ebebe9cea, gen: 196 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.447827189Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: e0e277c1-557f-4785-b433-9086b41beda9, session_id: 7ddaf088-1563-42cd-bdb8-f91ebebe9cea, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.448243465Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [323, 317, 323, 322, 322, 322, 314, 321, 321, 323, 312, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.449177124Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.471206848Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:46.471267511Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.471351012Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: e0e277c1-557f-4785-b433-9086b41beda9, session_id: 7ddaf088-1563-42cd-bdb8-f91ebebe9cea, gen: 196 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.47137259Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: e0e277c1-557f-4785-b433-9086b41beda9, session_id: 7ddaf088-1563-42cd-bdb8-f91ebebe9cea, gen: 196 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.471393397Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.607150641Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:62209"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.607943846Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"uri":"/region-info","method":"GET","req_id":"2a4a7c82-f47a-49bf-a63b-7f2702227eef","remote_addr":"127.0.0.1:62209","local_addr":"0.0.0.0:12820","task":"repair","latency_us":156,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:51625","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.610925525Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:51625): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.61107594Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"4"} {"msg":"Connection request from cef70586-b2e1-4ebf-bade-82d32d41c632 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.611319913Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: cef70586-b2e1-4ebf-bade-82d32d41c632, session_id: abe05ca3-a0d0-44f0-9605-bde58a05030b, gen: 197 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.61134428Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: cef70586-b2e1-4ebf-bade-82d32d41c632, session_id: abe05ca3-a0d0-44f0-9605-bde58a05030b, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.611756466Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [323, 330, 325, 322, 322, 329, 330, 321, 321, 323, 312, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.612677277Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.614071607Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.615499734Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.617642884Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00A.copy\" to \"/var/tmp/test_repair/8820/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.61823703Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.618275476Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00A.replace\" to \"/var/tmp/test_repair/8820/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.618404122Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.619741388Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.620245194Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.621930383Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/005.copy\" to \"/var/tmp/test_repair/8820/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.622463326Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/005.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.622496762Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/005.replace\" to \"/var/tmp/test_repair/8820/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.622661455Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.624034827Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.624539063Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.626165748Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.626734667Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.626759774Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:46.62689309Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:47.141890726Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:47.141959599Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:47.141986006Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.678901476Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:50.678972128Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.679000575Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: cef70586-b2e1-4ebf-bade-82d32d41c632, session_id: abe05ca3-a0d0-44f0-9605-bde58a05030b, gen: 197 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.679028292Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: cef70586-b2e1-4ebf-bade-82d32d41c632, session_id: abe05ca3-a0d0-44f0-9605-bde58a05030b, gen: 197 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.679054849Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection from 127.0.0.1:60967","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.778365329Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:60967): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.778415933Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"5"} {"msg":"Connection request from 36c28e01-258c-4d5b-8aa8-b1106ad4023f with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.778774575Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 36c28e01-258c-4d5b-8aa8-b1106ad4023f, session_id: 61512f4e-20b5-410c-a1cc-a7af11b95473, gen: 198 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.778799452Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: 36c28e01-258c-4d5b-8aa8-b1106ad4023f, session_id: 61512f4e-20b5-410c-a1cc-a7af11b95473, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.779209818Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [323, 330, 325, 322, 322, 331, 330, 321, 321, 323, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.78021491Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.799445714Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:50.799487749Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.799531344Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 36c28e01-258c-4d5b-8aa8-b1106ad4023f, session_id: 61512f4e-20b5-410c-a1cc-a7af11b95473, gen: 198 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.799554682Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: 36c28e01-258c-4d5b-8aa8-b1106ad4023f, session_id: 61512f4e-20b5-410c-a1cc-a7af11b95473, gen: 198 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.799576879Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"accepted connection from 127.0.0.1:33039","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.907283807Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"main"} {"msg":"connection (127.0.0.1:33039): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.907337931Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"id":"6"} {"msg":"Connection request from f1071cec-b754-442e-9aed-cd69462d5745 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.907615551Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: f1071cec-b754-442e-9aed-cd69462d5745, session_id: ad108874-8055-4df7-bc07-c7ab312e493d, gen: 199 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.907645758Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"UpstairsConnection { upstairs_id: f1071cec-b754-442e-9aed-cd69462d5745, session_id: ad108874-8055-4df7-bc07-c7ab312e493d, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.90809626Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Current flush_numbers [0..12]: [323, 333, 333, 333, 332, 331, 330, 333, 321, 323, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.908924451Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.910284415Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.910951123Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.912725402Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.913353675Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.913422327Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.913544965Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/013.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.915786584Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:19 Found repair files: [\"013\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.916430195Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 19 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.918087506Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/013.copy\" to \"/var/tmp/test_repair/8820/00/000/013.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.91870335Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/013.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.918733047Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/013.replace\" to \"/var/tmp/test_repair/8820/00/000/013.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.918903379Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.92019508Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.920796405Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.922214573Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.922824437Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.922859574Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.923024806Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.924306558Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.924961558Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.926499713Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/003.copy\" to \"/var/tmp/test_repair/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.927010578Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/003.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.927045954Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/003.replace\" to \"/var/tmp/test_repair/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.927168111Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.928520056Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.929146928Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.932604217Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.933085625Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.933127141Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.933245348Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.934763994Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.935438642Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.937018072Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.937558884Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.9375977Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.937772681Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.939116437Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.939604694Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.941122931Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.941614958Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.941646145Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.941773712Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.94318098Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.943673907Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.945284184Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/016.copy\" to \"/var/tmp/test_repair/8820/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.945775511Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/016.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.945807128Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/016.replace\" to \"/var/tmp/test_repair/8820/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.945939334Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.947270431Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.94792869Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.949686761Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/006.copy\" to \"/var/tmp/test_repair/8820/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.950181468Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/006.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.950208435Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/006.replace\" to \"/var/tmp/test_repair/8820/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:50.950326882Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464642362Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464708785Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464737322Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.46476314Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464789157Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 19 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464814334Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464839701Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464868868Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:51.464888406Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:55.008074542Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-23T18:55:55.008143424Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:55.00818742Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: f1071cec-b754-442e-9aed-cd69462d5745, session_id: ad108874-8055-4df7-bc07-c7ab312e493d, gen: 199 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:55.008241874Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954} {"msg":"upstairs UpstairsConnection { upstairs_id: f1071cec-b754-442e-9aed-cd69462d5745, session_id: ad108874-8055-4df7-bc07-c7ab312e493d, gen: 199 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-23T18:55:55.008269011Z","hostname":"w-01JJA7SA4NMW3BSXCPCAM7R9Y4","pid":1954}