{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.543604603Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.544337691Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.544362767Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.547646034Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.547675429Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7ae555c374784c668297ef5478b8ff4aaa8d57b4\nCommit timestamp: 2025-02-21T06:29:49.000000000Z branch: renovate/rust-1.x\nrustc: 1.85.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.59055022Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.590583165Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.591479466Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.591702179Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.591822299Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.591855644Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.592107322Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"accepted connection from 127.0.0.1:53132","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.592144106Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"connection (127.0.0.1:53132): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.592217374Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"id":"0"} {"msg":"Connection request from 438abdd4-62ac-44c7-abdb-36a63bcf7620 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.592581754Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 438abdd4-62ac-44c7-abdb-36a63bcf7620, session_id: 38992a92-91d6-4cc9-b9fe-21b4483d70a6, gen: 195 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.592609349Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UpstairsConnection { upstairs_id: 438abdd4-62ac-44c7-abdb-36a63bcf7620, session_id: 38992a92-91d6-4cc9-b9fe-21b4483d70a6, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.593035938Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Current flush_numbers [0..12]: [291, 313, 318, 318, 317, 305, 317, 318, 317, 312, 302, 315]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.593737752Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.59514063Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.596617535Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.598559524Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.599157365Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.599229463Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.599350103Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.600686812Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.601406762Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.603323375Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00C.copy\" to \"/var/tmp/test_repair/8820/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.603860126Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.603927485Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00C.replace\" to \"/var/tmp/test_repair/8820/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.604067192Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.605320585Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.605808044Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.607484246Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/019.copy\" to \"/var/tmp/test_repair/8820/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.60800381Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/019.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.608084847Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/019.replace\" to \"/var/tmp/test_repair/8820/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.608197508Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.609678843Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.610196587Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.611713056Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.61223205Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.612257456Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.612378366Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.613716204Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.614405Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.616081933Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/006.copy\" to \"/var/tmp/test_repair/8820/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.616594128Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/006.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.616624903Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/006.replace\" to \"/var/tmp/test_repair/8820/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.616744063Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.618019402Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.618560742Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.620196102Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/005.copy\" to \"/var/tmp/test_repair/8820/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.620697808Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/005.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.620723184Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/005.replace\" to \"/var/tmp/test_repair/8820/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.620849034Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/013.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.622171004Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:19 Found repair files: [\"013\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.626317988Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 19 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.628158693Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/013.copy\" to \"/var/tmp/test_repair/8820/00/000/013.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.628763593Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/013.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.628800707Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/013.replace\" to \"/var/tmp/test_repair/8820/00/000/013.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.628914578Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.63023134Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.63071328Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.632216601Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00D.copy\" to \"/var/tmp/test_repair/8820/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.632721598Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.632753432Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00D.replace\" to \"/var/tmp/test_repair/8820/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:40.632865144Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147678068Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147713222Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 12 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147737268Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147755675Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 19 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147771122Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147792229Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147813155Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:41.147835532Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.68297282Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:44.68303093Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.683063215Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 438abdd4-62ac-44c7-abdb-36a63bcf7620, session_id: 38992a92-91d6-4cc9-b9fe-21b4483d70a6, gen: 195 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.683088101Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 438abdd4-62ac-44c7-abdb-36a63bcf7620, session_id: 38992a92-91d6-4cc9-b9fe-21b4483d70a6, gen: 195 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.683103098Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"accepted connection from 127.0.0.1:54487","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.783159862Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"connection (127.0.0.1:54487): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.783210833Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"id":"1"} {"msg":"Connection request from 5e51e262-3378-412d-8963-49adc6110eb9 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.783574123Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e51e262-3378-412d-8963-49adc6110eb9, session_id: 2697a561-4b19-483b-867e-f362db26c483, gen: 196 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.783609477Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UpstairsConnection { upstairs_id: 5e51e262-3378-412d-8963-49adc6110eb9, session_id: 2697a561-4b19-483b-867e-f362db26c483, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.784021979Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Current flush_numbers [0..12]: [291, 313, 318, 318, 317, 320, 320, 318, 317, 312, 302, 315]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:44.784935178Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.806727581Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:54.806755676Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.806775633Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e51e262-3378-412d-8963-49adc6110eb9, session_id: 2697a561-4b19-483b-867e-f362db26c483, gen: 196 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.806818506Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e51e262-3378-412d-8963-49adc6110eb9, session_id: 2697a561-4b19-483b-867e-f362db26c483, gen: 196 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.806840922Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"accepted connection from 127.0.0.1:42720","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.914556218Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"connection (127.0.0.1:42720): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.914618647Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"id":"2"} {"msg":"Connection request from 101b515c-fc53-41b3-9739-816f652118eb with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.915011992Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 101b515c-fc53-41b3-9739-816f652118eb, session_id: 59a8cf30-7af0-45d5-8f22-67f809eaff7d, gen: 197 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.915052806Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UpstairsConnection { upstairs_id: 101b515c-fc53-41b3-9739-816f652118eb, session_id: 59a8cf30-7af0-45d5-8f22-67f809eaff7d, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.915454159Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Current flush_numbers [0..12]: [291, 313, 318, 318, 317, 320, 320, 318, 317, 312, 302, 321]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.916341812Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.91786193Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.918543098Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.920531419Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.921129819Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.921240751Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.92136709Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.923639334Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.924131412Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.925604799Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.92613701Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.926168945Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.926286566Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.927916006Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.928631987Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.930453666Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.930984868Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.931041768Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.93115225Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.932503666Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.933032509Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.934683855Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.935283056Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.935308022Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.935466336Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.936757752Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.93743167Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.939151206Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.939659161Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.939692806Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.939823434Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.941139447Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.941805566Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.943517353Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.943998853Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.944036827Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.944123652Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.945443954Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.94595317Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.947674944Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.94818326Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.948218524Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.948374449Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.94963374Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.950271204Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.951839845Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00A.copy\" to \"/var/tmp/test_repair/8820/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.952496246Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.952536729Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00A.replace\" to \"/var/tmp/test_repair/8820/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.952637513Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.953943367Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.954537658Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.956181256Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.956721576Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.956776887Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.956903246Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.958275459Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.958809191Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.960562091Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.961147564Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.961195386Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.961298759Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.962595934Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.963086473Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.964778433Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/009.copy\" to \"/var/tmp/test_repair/8820/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.965245575Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/009.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.965283329Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/009.replace\" to \"/var/tmp/test_repair/8820/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.965477997Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.967031849Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.967514729Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.969242934Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.96974779Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.969782114Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:54.969925721Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.484909057Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.484972076Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 9 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.484998222Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485021658Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485043844Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.48506851Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485094106Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485118632Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485143468Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485168374Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485194159Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:55.485220295Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.022785331Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.02285046Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.022873786Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 101b515c-fc53-41b3-9739-816f652118eb, session_id: 59a8cf30-7af0-45d5-8f22-67f809eaff7d, gen: 197 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.022896252Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: 101b515c-fc53-41b3-9739-816f652118eb, session_id: 59a8cf30-7af0-45d5-8f22-67f809eaff7d, gen: 197 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.022916509Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.152616385Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:37128"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.153448987Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"uri":"/region-info","method":"GET","req_id":"cd45c7ea-79d8-4f4f-b82e-6231b701f5bd","remote_addr":"127.0.0.1:37128","local_addr":"0.0.0.0:12820","task":"repair","latency_us":221,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:63692","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.15614879Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"connection (127.0.0.1:63692): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.156180265Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"id":"3"} {"msg":"Connection request from da07058d-5807-4dbd-a470-0ac2d60014ca with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.156620492Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: da07058d-5807-4dbd-a470-0ac2d60014ca, session_id: a42c2fb8-38a6-4ca1-b3b3-2f5df5cffb8d, gen: 198 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.156656946Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UpstairsConnection { upstairs_id: da07058d-5807-4dbd-a470-0ac2d60014ca, session_id: a42c2fb8-38a6-4ca1-b3b3-2f5df5cffb8d, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.157073717Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Current flush_numbers [0..12]: [291, 313, 322, 318, 317, 320, 320, 318, 322, 322, 322, 322]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.15796037Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.179749933Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.179785587Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.179812523Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: da07058d-5807-4dbd-a470-0ac2d60014ca, session_id: a42c2fb8-38a6-4ca1-b3b3-2f5df5cffb8d, gen: 198 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.179836199Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: da07058d-5807-4dbd-a470-0ac2d60014ca, session_id: a42c2fb8-38a6-4ca1-b3b3-2f5df5cffb8d, gen: 198 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.179856525Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.304621178Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:55760"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.305307955Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"uri":"/region-info","method":"GET","req_id":"22e05baf-b4b0-4fd1-878e-62d0d3497d75","remote_addr":"127.0.0.1:55760","local_addr":"0.0.0.0:12820","task":"repair","latency_us":124,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:47049","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.307866851Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"main"} {"msg":"connection (127.0.0.1:47049): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.307971804Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"id":"4"} {"msg":"Connection request from a8a822f7-6eea-49ba-8a6c-696c3d25601d with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.308411871Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: a8a822f7-6eea-49ba-8a6c-696c3d25601d, session_id: 6ecd6885-52cb-4dd5-8af5-27e5d6f2ed13, gen: 199 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.308447365Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"UpstairsConnection { upstairs_id: a8a822f7-6eea-49ba-8a6c-696c3d25601d, session_id: 6ecd6885-52cb-4dd5-8af5-27e5d6f2ed13, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.308871475Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Current flush_numbers [0..12]: [291, 313, 323, 318, 317, 320, 320, 318, 325, 322, 322, 322]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.309662753Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.31107336Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.312481617Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.31463883Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.318761997Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.3188034Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.318962464Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.321749152Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.322256119Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.32405917Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.324555248Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.324589892Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.324697064Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.326217143Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.326659339Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.328275192Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.328784998Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.328818332Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.329029767Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.330496154Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.330970286Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.33245616Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/015.copy\" to \"/var/tmp/test_repair/8820/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.332965175Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/015.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.333016067Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/015.replace\" to \"/var/tmp/test_repair/8820/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.333131018Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.33474678Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.335405482Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.337041231Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.337605877Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.33764826Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:52:59.337783438Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.851350348Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.851416737Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.851440593Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.8514625Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:52:59.851491895Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:53:03.387836774Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-21T06:53:03.387907172Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:53:03.387935727Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: a8a822f7-6eea-49ba-8a6c-696c3d25601d, session_id: 6ecd6885-52cb-4dd5-8af5-27e5d6f2ed13, gen: 199 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:53:03.387961473Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971} {"msg":"upstairs UpstairsConnection { upstairs_id: a8a822f7-6eea-49ba-8a6c-696c3d25601d, session_id: 6ecd6885-52cb-4dd5-8af5-27e5d6f2ed13, gen: 199 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-21T06:53:03.387989868Z","hostname":"w-01JMKKZ435WR2BFDX5TYESJR88","pid":1971}