{"msg":"Opened existing region file \"/var/tmp/test_repair/8830/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.072138141Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.072574836Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.07259907Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"UUID: 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.075636405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.075665502Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: abd5430f1508af0bc008143ea2566277ef714ec2\nCommit timestamp: 2025-01-17T20:55:03.000000000Z branch: alan/tokio-to-140\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.118838456Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.11889692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12830 for path:\"/var/tmp/test_repair/8830\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.119819079Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.120164901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"local_addr":"0.0.0.0:12830","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.120296848Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.120325705Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8830","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.120364568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"accepted connection from 127.0.0.1:61824","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.121690372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"connection (127.0.0.1:61824): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.121830485Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"id":"0"} {"msg":"Connection request from 5df0043b-e140-40e1-826d-2beef3d2c0a2 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.122064491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.122091407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.122571628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Current flush_numbers [0..12]: [349, 348, 346, 347, 352, 350, 350, 350, 352, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.123352354Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.124860115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.126637094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.128572665Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00E.copy\" to \"/var/tmp/test_repair/8830/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.129123857Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00E.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.129175998Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00E.replace\" to \"/var/tmp/test_repair/8830/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.129303102Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.130704021Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.13124997Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133082971Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/016.copy\" to \"/var/tmp/test_repair/8830/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133655536Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/016.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133688565Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/016.replace\" to \"/var/tmp/test_repair/8830/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133860096Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.135190053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.135710607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137365644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/001.copy\" to \"/var/tmp/test_repair/8830/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137897365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/001.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137926672Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/001.replace\" to \"/var/tmp/test_repair/8830/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.138054417Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.139371607Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.139933215Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.141748396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/002.copy\" to \"/var/tmp/test_repair/8830/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.142252911Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/002.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.142318739Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/002.replace\" to \"/var/tmp/test_repair/8830/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.14243962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.143839178Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.146292172Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148023213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00A.copy\" to \"/var/tmp/test_repair/8830/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148610877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00A.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148646167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00A.replace\" to \"/var/tmp/test_repair/8830/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148750328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.150311501Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.15082349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.152579136Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/000.copy\" to \"/var/tmp/test_repair/8830/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.153096819Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/000.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.153137182Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/000.replace\" to \"/var/tmp/test_repair/8830/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.153258703Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669351605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669392669Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669416253Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669449142Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669475828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669503724Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208465155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.20859463Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208623207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208638276Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208652104Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"accepted connection from 127.0.0.1:59645","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342325403Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"connection (127.0.0.1:59645): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342409772Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"id":"1"} {"msg":"Connection request from c9f591c0-4528-4e75-9b2a-fad223b5379e with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.34280092Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342836591Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343252694Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Current flush_numbers [0..12]: [353, 353, 353, 347, 352, 350, 350, 350, 352, 345, 353, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344207262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366403936Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.366441267Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366466692Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366491647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366521805Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"accepted connection from 127.0.0.1:64204","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.498859622Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"main"} {"msg":"connection (127.0.0.1:64204): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.498947934Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"id":"2"} {"msg":"Connection request from 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.499387101Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.499418059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.499841626Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Current flush_numbers [0..12]: [354, 354, 353, 347, 352, 350, 354, 350, 352, 345, 354, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.50063435Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.502135767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.503484555Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.505479811Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/001.copy\" to \"/var/tmp/test_repair/8830/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.506145311Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/001.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.50616084Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/001.replace\" to \"/var/tmp/test_repair/8830/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.506272695Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.50763256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.508105966Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.509814605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00B.copy\" to \"/var/tmp/test_repair/8830/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510377514Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00B.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510402839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00B.replace\" to \"/var/tmp/test_repair/8830/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510527402Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.512211876Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.512742877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.514468865Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00E.copy\" to \"/var/tmp/test_repair/8830/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.514976372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00E.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.515008781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00E.replace\" to \"/var/tmp/test_repair/8830/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.515137036Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.516536314Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.51700993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.518698828Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/017.copy\" to \"/var/tmp/test_repair/8830/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.519229048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/017.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.519261597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/017.replace\" to \"/var/tmp/test_repair/8830/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.519457771Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.523317407Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.523817199Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.525692575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00F.copy\" to \"/var/tmp/test_repair/8830/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526168053Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00F.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526199152Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00F.replace\" to \"/var/tmp/test_repair/8830/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526340714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.52768504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.528192586Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.52984004Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00A.copy\" to \"/var/tmp/test_repair/8830/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530379155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00A.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530429434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00A.replace\" to \"/var/tmp/test_repair/8830/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530544491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.532092696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.532635754Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.534724984Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/008.copy\" to \"/var/tmp/test_repair/8830/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.535248891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/008.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.535279499Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/008.replace\" to \"/var/tmp/test_repair/8830/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.535403721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049724767Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.04978132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049825156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.04986688Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049890504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049912396Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049938862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.587909064Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:41.587982988Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.58800491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.588025993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.588046815Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1968}