{"msg":"Opened existing region file \"/var/tmp/test_repair/8830/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.740937378Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.741515993Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.741541979Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UUID: 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.744264879Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.744295424Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7817632486f84ab6865c5ce71e16196ec0c710fc\nCommit timestamp: 2025-01-28T21:38:04.000000000Z branch: mkeeter/early-io-rejection\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.788358999Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.78842107Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12830 for path:\"/var/tmp/test_repair/8830\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.789385078Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.789681315Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"local_addr":"0.0.0.0:12830","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.789804507Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.789839922Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.789884745Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"accepted connection from 127.0.0.1:35169","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.791739523Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:35169): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.791965949Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"0"} {"msg":"Connection request from 7eb223d4-d30f-43e4-a3b2-427c1147eec8 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.79216353Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 7eb223d4-d30f-43e4-a3b2-427c1147eec8, session_id: 79cbfb3f-3e49-4129-ab3c-df2f54b28115, gen: 187 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.792191846Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 7eb223d4-d30f-43e4-a3b2-427c1147eec8, session_id: 79cbfb3f-3e49-4129-ab3c-df2f54b28115, gen: 187 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.792781709Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [328, 333, 312, 331, 332, 313, 327, 331, 330, 332, 309, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.793667699Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.794973117Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.796737378Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.798485491Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/011.copy\" to \"/var/tmp/test_repair/8830/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.798998546Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/011.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.799069386Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/011.replace\" to \"/var/tmp/test_repair/8830/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.799198767Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/018.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.800512094Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:24 Found repair files: [\"018\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.801037716Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 24 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.802623193Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/018.copy\" to \"/var/tmp/test_repair/8830/00/000/018.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.803162024Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/018.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.803205838Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/018.replace\" to \"/var/tmp/test_repair/8830/00/000/018.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.80332267Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.804705927Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.80523101Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.811594925Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/010.copy\" to \"/var/tmp/test_repair/8830/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.812099851Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/010.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.812131746Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/010.replace\" to \"/var/tmp/test_repair/8830/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.812252388Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.813586912Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.814086479Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.815742765Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/001.copy\" to \"/var/tmp/test_repair/8830/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.816243632Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/001.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.816272328Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/001.replace\" to \"/var/tmp/test_repair/8830/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.816410787Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.817869553Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.818362191Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.820095016Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01D.copy\" to \"/var/tmp/test_repair/8830/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.820628987Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01D.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.820661543Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01D.replace\" to \"/var/tmp/test_repair/8830/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.82081368Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.822180119Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.822685615Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.824190714Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/015.copy\" to \"/var/tmp/test_repair/8830/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.824708758Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/015.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.824752461Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/015.replace\" to \"/var/tmp/test_repair/8830/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:03:56.824905799Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.339824616Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.339869189Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.339897335Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.339920272Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 24 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.33993513Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:03:57.339949488Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:00.881476127Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:00.88165487Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:00.881696824Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 7eb223d4-d30f-43e4-a3b2-427c1147eec8, session_id: 79cbfb3f-3e49-4129-ab3c-df2f54b28115, gen: 187 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:00.881720441Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 7eb223d4-d30f-43e4-a3b2-427c1147eec8, session_id: 79cbfb3f-3e49-4129-ab3c-df2f54b28115, gen: 187 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:00.881743177Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:52689","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.020400173Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:52689): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.020459564Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"1"} {"msg":"Connection request from 85a6432b-c2f9-4d9c-bb5d-50244be5bf69 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.021087022Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 85a6432b-c2f9-4d9c-bb5d-50244be5bf69, session_id: a8187c19-30ab-4233-b806-ee15deb32702, gen: 188 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.021115368Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 85a6432b-c2f9-4d9c-bb5d-50244be5bf69, session_id: a8187c19-30ab-4233-b806-ee15deb32702, gen: 188 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.021536326Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [328, 334, 312, 331, 332, 313, 327, 331, 330, 332, 309, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.022489856Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.042273759Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.042312623Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.042368275Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 85a6432b-c2f9-4d9c-bb5d-50244be5bf69, session_id: a8187c19-30ab-4233-b806-ee15deb32702, gen: 188 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.04240118Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 85a6432b-c2f9-4d9c-bb5d-50244be5bf69, session_id: a8187c19-30ab-4233-b806-ee15deb32702, gen: 188 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.042431396Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:48952","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.176383932Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:48952): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.176506784Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"2"} {"msg":"Connection request from 26e96705-8188-4321-aec5-98e8b9671d20 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.176981095Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 26e96705-8188-4321-aec5-98e8b9671d20, session_id: bdc66827-5122-4d8d-bf32-4b2bbfea9549, gen: 189 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.177049435Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 26e96705-8188-4321-aec5-98e8b9671d20, session_id: bdc66827-5122-4d8d-bf32-4b2bbfea9549, gen: 189 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.177456115Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [335, 335, 335, 331, 335, 313, 327, 331, 330, 335, 335, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.178474005Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.179865081Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.18123556Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.183327382Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00D.copy\" to \"/var/tmp/test_repair/8830/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.183948751Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00D.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.18402334Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00D.replace\" to \"/var/tmp/test_repair/8830/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.184173768Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.185492494Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.186052482Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.187925406Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01C.copy\" to \"/var/tmp/test_repair/8830/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.188467877Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01C.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.188524329Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01C.replace\" to \"/var/tmp/test_repair/8830/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.188679696Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.189972056Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.190509407Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.192236593Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/002.copy\" to \"/var/tmp/test_repair/8830/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.192743498Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/002.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.192779283Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/002.replace\" to \"/var/tmp/test_repair/8830/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.192925362Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/013.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.194371389Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:19 Found repair files: [\"013\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.19491113Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 19 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.196633167Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/013.copy\" to \"/var/tmp/test_repair/8830/00/000/013.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.197188555Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/013.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.197213792Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/013.replace\" to \"/var/tmp/test_repair/8830/00/000/013.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.197340363Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.198779012Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:27 Found repair files: [\"01B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.19926364Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 27 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.200824391Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01B.copy\" to \"/var/tmp/test_repair/8830/00/000/01B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.201365931Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01B.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.201424993Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01B.replace\" to \"/var/tmp/test_repair/8830/00/000/01B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:01.201617285Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.718444891Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.718513431Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 19 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.718557194Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 27 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.718595329Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:01.718622365Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.25923295Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:05.25930009Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.259327506Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 26e96705-8188-4321-aec5-98e8b9671d20, session_id: bdc66827-5122-4d8d-bf32-4b2bbfea9549, gen: 189 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.259354862Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 26e96705-8188-4321-aec5-98e8b9671d20, session_id: bdc66827-5122-4d8d-bf32-4b2bbfea9549, gen: 189 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.259382428Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:65301","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.363919557Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:65301): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.364015573Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"3"} {"msg":"Connection request from e5020bc5-ea67-469d-b2a3-3c25d8857e9a with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.364393078Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: e5020bc5-ea67-469d-b2a3-3c25d8857e9a, session_id: fa25cedd-05fe-439c-b96d-503a011e1201, gen: 190 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.364429382Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: e5020bc5-ea67-469d-b2a3-3c25d8857e9a, session_id: fa25cedd-05fe-439c-b96d-503a011e1201, gen: 190 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.364883056Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [335, 335, 337, 331, 335, 313, 327, 331, 330, 335, 335, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.365850953Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.387730478Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:05.387772132Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.387798379Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: e5020bc5-ea67-469d-b2a3-3c25d8857e9a, session_id: fa25cedd-05fe-439c-b96d-503a011e1201, gen: 190 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.387820215Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: e5020bc5-ea67-469d-b2a3-3c25d8857e9a, session_id: fa25cedd-05fe-439c-b96d-503a011e1201, gen: 190 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.387841722Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:48431","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.489243832Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:48431): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.489303273Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"4"} {"msg":"Connection request from 3974025a-fea5-4227-a6ac-d89e5b343ea7 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.489853822Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 3974025a-fea5-4227-a6ac-d89e5b343ea7, session_id: e4ed9049-c692-4315-961b-7a41a2253b7c, gen: 191 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.489887327Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 3974025a-fea5-4227-a6ac-d89e5b343ea7, session_id: e4ed9049-c692-4315-961b-7a41a2253b7c, gen: 191 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.490334482Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [335, 335, 337, 331, 335, 313, 338, 338, 342, 342, 339, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.491245378Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.492530709Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.493178244Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.494940705Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/006.copy\" to \"/var/tmp/test_repair/8830/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.49551823Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/006.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.495594239Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/006.replace\" to \"/var/tmp/test_repair/8830/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.495784211Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.49803883Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.498562923Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.500166487Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/012.copy\" to \"/var/tmp/test_repair/8830/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.500699909Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/012.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.500748562Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/012.replace\" to \"/var/tmp/test_repair/8830/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:05.500878152Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:06.015262887Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:06.015315629Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.555786195Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:09.555851706Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.555878822Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 3974025a-fea5-4227-a6ac-d89e5b343ea7, session_id: e4ed9049-c692-4315-961b-7a41a2253b7c, gen: 191 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.555905888Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 3974025a-fea5-4227-a6ac-d89e5b343ea7, session_id: e4ed9049-c692-4315-961b-7a41a2253b7c, gen: 191 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.555932264Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:61840","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.692020317Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:61840): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.692107104Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"5"} {"msg":"Connection request from 13101720-6ed9-452e-b266-25073f7c406a with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.69247566Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 13101720-6ed9-452e-b266-25073f7c406a, session_id: 0c7085e7-e6f5-46c9-903d-ad8bd8d23cd3, gen: 192 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.692511475Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 13101720-6ed9-452e-b266-25073f7c406a, session_id: 0c7085e7-e6f5-46c9-903d-ad8bd8d23cd3, gen: 192 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.692919515Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [335, 335, 337, 331, 335, 313, 343, 338, 342, 342, 339, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.693730985Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.714007306Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:09.714040761Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.714062748Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 13101720-6ed9-452e-b266-25073f7c406a, session_id: 0c7085e7-e6f5-46c9-903d-ad8bd8d23cd3, gen: 192 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.714084605Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 13101720-6ed9-452e-b266-25073f7c406a, session_id: 0c7085e7-e6f5-46c9-903d-ad8bd8d23cd3, gen: 192 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.714106131Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:49489","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.845848292Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:49489): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.845946538Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"6"} {"msg":"Connection request from 81172c51-5c71-40e6-93ed-ac16f4b23802 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.846372685Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 81172c51-5c71-40e6-93ed-ac16f4b23802, session_id: 48260206-f745-448d-ae22-b4a929875227, gen: 193 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.846397352Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 81172c51-5c71-40e6-93ed-ac16f4b23802, session_id: 48260206-f745-448d-ae22-b4a929875227, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.846805432Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [345, 345, 337, 344, 344, 313, 343, 338, 342, 342, 344, 347]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.847832691Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.849221667Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.850641608Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.852763086Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00A.copy\" to \"/var/tmp/test_repair/8830/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.853402632Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00A.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.853445136Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00A.replace\" to \"/var/tmp/test_repair/8830/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.853584555Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.854867957Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.855398429Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.85702733Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/005.copy\" to \"/var/tmp/test_repair/8830/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.85770388Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/005.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.857747574Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/005.replace\" to \"/var/tmp/test_repair/8830/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.85790876Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.859179353Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.859671471Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.861227563Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/009.copy\" to \"/var/tmp/test_repair/8830/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.861746656Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/009.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.861778702Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/009.replace\" to \"/var/tmp/test_repair/8830/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:09.861899384Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:10.376386554Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 9 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:10.376462413Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:10.376485699Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:13.916950596Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:13.916994709Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:13.917015896Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 81172c51-5c71-40e6-93ed-ac16f4b23802, session_id: 48260206-f745-448d-ae22-b4a929875227, gen: 193 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:13.917037443Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 81172c51-5c71-40e6-93ed-ac16f4b23802, session_id: 48260206-f745-448d-ae22-b4a929875227, gen: 193 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:13.917095245Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:53408","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.054029023Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:53408): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.054141027Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"7"} {"msg":"Connection request from c8e25ff8-0ecc-414b-bad3-d85ea02b5ba6 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.054543577Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: c8e25ff8-0ecc-414b-bad3-d85ea02b5ba6, session_id: b3f72404-61ef-4159-8e83-aa9662f0507d, gen: 194 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.054578203Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: c8e25ff8-0ecc-414b-bad3-d85ea02b5ba6, session_id: b3f72404-61ef-4159-8e83-aa9662f0507d, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.055018368Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [345, 345, 337, 344, 344, 348, 343, 338, 342, 348, 348, 347]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.055971108Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.076274214Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"7"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.076311599Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"7"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.076336045Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: c8e25ff8-0ecc-414b-bad3-d85ea02b5ba6, session_id: b3f72404-61ef-4159-8e83-aa9662f0507d, gen: 194 } (ConnectionId(7)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.076358002Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: c8e25ff8-0ecc-414b-bad3-d85ea02b5ba6, session_id: b3f72404-61ef-4159-8e83-aa9662f0507d, gen: 194 } (ConnectionId(7)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.076379959Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:34357","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.212288368Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:34357): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.212338971Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"8"} {"msg":"Connection request from 6c984011-5caa-4ded-b431-d1c60cb5731b with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.212768468Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 6c984011-5caa-4ded-b431-d1c60cb5731b, session_id: 14d6f2a7-d56c-4870-9ec8-4af32cc1bdb9, gen: 195 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.212796534Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 6c984011-5caa-4ded-b431-d1c60cb5731b, session_id: 14d6f2a7-d56c-4870-9ec8-4af32cc1bdb9, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.213245608Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [349, 345, 350, 350, 344, 350, 349, 338, 350, 350, 349, 349]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.214191538Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.215444465Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.216781068Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.218788643Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/001.copy\" to \"/var/tmp/test_repair/8830/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.219344501Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/001.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.219380386Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/001.replace\" to \"/var/tmp/test_repair/8830/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.219508367Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.221029574Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.221518442Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.223267525Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/004.copy\" to \"/var/tmp/test_repair/8830/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.223782959Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/004.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.223808565Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/004.replace\" to \"/var/tmp/test_repair/8830/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.223954914Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.225535462Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.226076892Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.227670658Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/000.copy\" to \"/var/tmp/test_repair/8830/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.228176554Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/000.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.228230186Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/000.replace\" to \"/var/tmp/test_repair/8830/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.228420198Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.229868815Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.234570424Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.236148462Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/006.copy\" to \"/var/tmp/test_repair/8830/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.236668666Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/006.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.236698181Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/006.replace\" to \"/var/tmp/test_repair/8830/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.236823733Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.238178004Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.238657903Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.240265477Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/003.copy\" to \"/var/tmp/test_repair/8830/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.240761314Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/003.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.24079073Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/003.replace\" to \"/var/tmp/test_repair/8830/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.240914542Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.242441187Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.242926976Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.244484257Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01A.copy\" to \"/var/tmp/test_repair/8830/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.245039706Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01A.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.24508225Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01A.replace\" to \"/var/tmp/test_repair/8830/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.245194703Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.246971262Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.247651732Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.249525146Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/014.copy\" to \"/var/tmp/test_repair/8830/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.250034542Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/014.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.250061118Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/014.replace\" to \"/var/tmp/test_repair/8830/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.25018038Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.251480719Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.251960319Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.253814276Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/007.copy\" to \"/var/tmp/test_repair/8830/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.254432405Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/007.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.25447192Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/007.replace\" to \"/var/tmp/test_repair/8830/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.25460617Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.25590046Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.256466117Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.258029387Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/005.copy\" to \"/var/tmp/test_repair/8830/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.26113855Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/005.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.261169335Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/005.replace\" to \"/var/tmp/test_repair/8830/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:14.261310705Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775738373Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775838208Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775868404Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775890101Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775912437Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775934274Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775956221Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.775978358Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:14.776001094Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.317604794Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"8"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:18.317657586Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"8"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.317679293Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 6c984011-5caa-4ded-b431-d1c60cb5731b, session_id: 14d6f2a7-d56c-4870-9ec8-4af32cc1bdb9, gen: 195 } (ConnectionId(8)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.31770309Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 6c984011-5caa-4ded-b431-d1c60cb5731b, session_id: 14d6f2a7-d56c-4870-9ec8-4af32cc1bdb9, gen: 195 } (ConnectionId(8)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.317725246Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:53922","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.42413875Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:53922): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.424228816Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"9"} {"msg":"Connection request from ac96a292-7115-4d02-b51b-dc64b521c1fe with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.424526263Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ac96a292-7115-4d02-b51b-dc64b521c1fe, session_id: 2c30b198-1e39-4e90-b1a0-547eadff6ba0, gen: 196 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.424559008Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: ac96a292-7115-4d02-b51b-dc64b521c1fe, session_id: 2c30b198-1e39-4e90-b1a0-547eadff6ba0, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.424958959Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [351, 351, 350, 351, 351, 351, 351, 351, 350, 350, 349, 349]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.425691362Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.445139534Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"9"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:18.445176768Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"9"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.445202595Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ac96a292-7115-4d02-b51b-dc64b521c1fe, session_id: 2c30b198-1e39-4e90-b1a0-547eadff6ba0, gen: 196 } (ConnectionId(9)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.445224581Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ac96a292-7115-4d02-b51b-dc64b521c1fe, session_id: 2c30b198-1e39-4e90-b1a0-547eadff6ba0, gen: 196 } (ConnectionId(9)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.445246368Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:62934","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.546071982Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:62934): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.546108797Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"10"} {"msg":"Connection request from 4537ac38-3426-4f06-aae9-30150f39e3d1 with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.546530835Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 4537ac38-3426-4f06-aae9-30150f39e3d1, session_id: 69f0cd0a-76f8-44be-8b7e-5d789e653893, gen: 197 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.546569929Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 4537ac38-3426-4f06-aae9-30150f39e3d1, session_id: 69f0cd0a-76f8-44be-8b7e-5d789e653893, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.546965991Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [352, 351, 350, 351, 351, 352, 351, 351, 350, 350, 352, 352]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.547986101Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.549197183Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.549804094Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.551662101Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01D.copy\" to \"/var/tmp/test_repair/8830/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.552249425Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01D.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.552292819Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01D.replace\" to \"/var/tmp/test_repair/8830/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.552439767Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.555023767Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.555529913Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.557196718Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/006.copy\" to \"/var/tmp/test_repair/8830/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.557716382Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/006.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.557742808Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/006.replace\" to \"/var/tmp/test_repair/8830/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.557869919Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.559241557Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.559787497Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.561362716Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/00B.copy\" to \"/var/tmp/test_repair/8830/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.561974956Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/00B.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.56201523Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/00B.replace\" to \"/var/tmp/test_repair/8830/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:18.562168037Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:19.076655847Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:19.076718288Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:19.076741995Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.616006377Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"10"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:22.616102693Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"10"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.616129219Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 4537ac38-3426-4f06-aae9-30150f39e3d1, session_id: 69f0cd0a-76f8-44be-8b7e-5d789e653893, gen: 197 } (ConnectionId(10)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.616146947Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 4537ac38-3426-4f06-aae9-30150f39e3d1, session_id: 69f0cd0a-76f8-44be-8b7e-5d789e653893, gen: 197 } (ConnectionId(10)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.616164304Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:41812","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.72085893Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:41812): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.720918981Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"11"} {"msg":"Connection request from ea8d6bf5-b38c-4b7a-a443-1ca32de841fb with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.721271989Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ea8d6bf5-b38c-4b7a-a443-1ca32de841fb, session_id: 8f9e220c-27ca-4a45-a399-3ac87da8410a, gen: 198 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.721299435Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: ea8d6bf5-b38c-4b7a-a443-1ca32de841fb, session_id: 8f9e220c-27ca-4a45-a399-3ac87da8410a, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.721697877Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [352, 351, 350, 351, 351, 352, 353, 351, 350, 350, 352, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.722485561Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.743304892Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"11"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:22.743368073Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"11"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.743403857Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ea8d6bf5-b38c-4b7a-a443-1ca32de841fb, session_id: 8f9e220c-27ca-4a45-a399-3ac87da8410a, gen: 198 } (ConnectionId(11)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.743432253Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: ea8d6bf5-b38c-4b7a-a443-1ca32de841fb, session_id: 8f9e220c-27ca-4a45-a399-3ac87da8410a, gen: 198 } (ConnectionId(11)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.743457349Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"accepted connection from 127.0.0.1:33167","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.844610846Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"main"} {"msg":"connection (127.0.0.1:33167): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.844701252Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"id":"12"} {"msg":"Connection request from 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.845150017Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.845187421Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.845548748Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Current flush_numbers [0..12]: [352, 354, 350, 355, 356, 352, 353, 351, 350, 356, 352, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.846558399Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.847784939Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.848371643Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.85016467Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/000.copy\" to \"/var/tmp/test_repair/8830/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.850810795Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/000.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.85084152Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/000.replace\" to \"/var/tmp/test_repair/8830/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.851036732Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.853505679Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.854008975Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.8558132Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/001.copy\" to \"/var/tmp/test_repair/8830/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856326214Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/001.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856362999Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/001.replace\" to \"/var/tmp/test_repair/8830/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856503948Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.858043822Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.858528631Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.860345334Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01D.copy\" to \"/var/tmp/test_repair/8830/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.862814511Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01D.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.862845216Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01D.replace\" to \"/var/tmp/test_repair/8830/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.862963889Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.864582171Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.86513125Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.86683529Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/019.copy\" to \"/var/tmp/test_repair/8830/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867357693Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/019.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867389958Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/019.replace\" to \"/var/tmp/test_repair/8830/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867527688Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381762016Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381818407Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381845863Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.38186934Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.92475257Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"12"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:26.924810452Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930,"task":"recv","id":"12"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.924839927Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } (ConnectionId(12)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.924867134Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } (ConnectionId(12)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.924895639Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1930}