{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.989228856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.989585524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.98961489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.99132706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.991354026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 94736ad28ed33108b1c88fccc3caea693da35514\nCommit timestamp: 2025-01-03T08:14:58.000000000Z branch: renovate/tempfile-3.x-lockfile\nrustc: 1.82.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.0346108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.034668952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.035657707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.036045581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.03611664Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.036140937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.036234643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"accepted connection from 127.0.0.1:64770","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.068735378Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:64770): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.068819835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"0"} {"msg":"Connection request from 5e1c70c8-74a0-4b07-9525-0580e26bc2bb with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.069341139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e1c70c8-74a0-4b07-9525-0580e26bc2bb, session_id: ea03a505-e002-4614-9d19-e124cc102961, gen: 191 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.069379524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: 5e1c70c8-74a0-4b07-9525-0580e26bc2bb, session_id: ea03a505-e002-4614-9d19-e124cc102961, gen: 191 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.069831238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [302, 299, 301, 302, 301, 291, 299, 299, 297, 301, 301, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.070645788Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.072098416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.073621214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.075556151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00A.copy\" to \"/var/tmp/test_repair/8820/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.076084485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.076171012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00A.replace\" to \"/var/tmp/test_repair/8820/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:42.076297393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:42.59127397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.131222407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:46.131372345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.131414399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e1c70c8-74a0-4b07-9525-0580e26bc2bb, session_id: ea03a505-e002-4614-9d19-e124cc102961, gen: 191 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.131447354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 5e1c70c8-74a0-4b07-9525-0580e26bc2bb, session_id: ea03a505-e002-4614-9d19-e124cc102961, gen: 191 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.131472081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.261403259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:51334"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.262168457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"972128c2-51f0-4dcf-8e86-3b4ee4afdd78","remote_addr":"127.0.0.1:51334","local_addr":"0.0.0.0:12820","task":"repair","latency_us":197,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:35715","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294553009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:35715): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294667432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"1"} {"msg":"Connection request from afe2039b-9394-4001-a881-1ef1fa7a71a4 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295001853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: afe2039b-9394-4001-a881-1ef1fa7a71a4, session_id: d56e5e1e-a95a-4f22-8d61-1cc7767232f7, gen: 192 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295032079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: afe2039b-9394-4001-a881-1ef1fa7a71a4, session_id: d56e5e1e-a95a-4f22-8d61-1cc7767232f7, gen: 192 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295445688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [302, 299, 301, 302, 301, 291, 299, 299, 297, 301, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296406548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.318540036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:46.31858326Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.318607196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: afe2039b-9394-4001-a881-1ef1fa7a71a4, session_id: d56e5e1e-a95a-4f22-8d61-1cc7767232f7, gen: 192 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.318628823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: afe2039b-9394-4001-a881-1ef1fa7a71a4, session_id: d56e5e1e-a95a-4f22-8d61-1cc7767232f7, gen: 192 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.3186509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.446919871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:50128"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.447669581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"de8dff97-4f37-471a-b8ef-7adeea7a6b36","remote_addr":"127.0.0.1:50128","local_addr":"0.0.0.0:12820","task":"repair","latency_us":132,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:34526","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.480001071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:34526): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.480044444Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"2"} {"msg":"Connection request from 8dcceba3-a574-41f0-be0f-7c46988820e0 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.480612661Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 8dcceba3-a574-41f0-be0f-7c46988820e0, session_id: 36eaae6a-8fe3-4211-a8ae-98dfc7ee7f27, gen: 193 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.480660224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: 8dcceba3-a574-41f0-be0f-7c46988820e0, session_id: 36eaae6a-8fe3-4211-a8ae-98dfc7ee7f27, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.481054397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [304, 299, 301, 302, 301, 291, 305, 305, 306, 301, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.481983691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.483627551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.485223008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.487090156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.487615529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.487661602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.487794593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.489342257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.489863971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.491700592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/017.copy\" to \"/var/tmp/test_repair/8820/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.492215687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/017.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.492251382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/017.replace\" to \"/var/tmp/test_repair/8820/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.492411108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.494026943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.494604098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.49623543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.496761803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.496790029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.496917001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.498508958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.499032252Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.500462133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/004.copy\" to \"/var/tmp/test_repair/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.501038609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/004.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.501093451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/004.replace\" to \"/var/tmp/test_repair/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.501264196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.502747609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.503445227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.508513377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/005.copy\" to \"/var/tmp/test_repair/8820/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.509037041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/005.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.509073066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/005.replace\" to \"/var/tmp/test_repair/8820/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.509211406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.511226161Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.511720209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.513422121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/006.copy\" to \"/var/tmp/test_repair/8820/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.513940345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/006.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.51397236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/006.replace\" to \"/var/tmp/test_repair/8820/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.514090263Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/018.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.515669263Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:24 Found repair files: [\"018\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.516228871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 24 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.518007901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/018.copy\" to \"/var/tmp/test_repair/8820/00/000/018.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.518532534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/018.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.518569109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/018.replace\" to \"/var/tmp/test_repair/8820/00/000/018.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.518685262Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.520018728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.520557109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.522003278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/015.copy\" to \"/var/tmp/test_repair/8820/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.522632916Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/015.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.522667111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/015.replace\" to \"/var/tmp/test_repair/8820/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.522791603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.524664569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.525190982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.526924089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/009.copy\" to \"/var/tmp/test_repair/8820/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.530519904Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/009.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.530591204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/009.replace\" to \"/var/tmp/test_repair/8820/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.530791695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.045876935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.045950004Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.04598225Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 9 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.046011995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.046058688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.046087474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.04611356Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.046141566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 24 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:47.046170982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.584495937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:50.584554298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.584586604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 8dcceba3-a574-41f0-be0f-7c46988820e0, session_id: 36eaae6a-8fe3-4211-a8ae-98dfc7ee7f27, gen: 193 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.58461403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 8dcceba3-a574-41f0-be0f-7c46988820e0, session_id: 36eaae6a-8fe3-4211-a8ae-98dfc7ee7f27, gen: 193 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.584646335Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.712463292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:39374"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.7131643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"09be2081-00f6-444c-9c01-920c698c3b3a","remote_addr":"127.0.0.1:39374","local_addr":"0.0.0.0:12820","task":"repair","latency_us":125,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:32982","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745909678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:32982): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745958271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"3"} {"msg":"Connection request from e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746279815Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95, session_id: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49, gen: 194 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746315999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95, session_id: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746715311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [304, 299, 301, 302, 308, 308, 308, 305, 306, 308, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747552689Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.768352082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:50.768390836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.768415892Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95, session_id: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49, gen: 194 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.768442768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95, session_id: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49, gen: 194 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.768469644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.899300261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:55165"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.900005648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"210a82b9-c266-4913-bdbd-fb23da463626","remote_addr":"127.0.0.1:55165","local_addr":"0.0.0.0:12820","task":"repair","latency_us":135,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:34934","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.933515165Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:34934): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.933575726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"4"} {"msg":"Connection request from d62ec780-2fa6-448e-96ab-533d8bfa2533 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.934017772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: d62ec780-2fa6-448e-96ab-533d8bfa2533, session_id: 59a37b07-a36a-4c2b-84f6-5bbe2e9e940f, gen: 195 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.934055656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: d62ec780-2fa6-448e-96ab-533d8bfa2533, session_id: 59a37b07-a36a-4c2b-84f6-5bbe2e9e940f, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.934545995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [309, 309, 309, 302, 308, 308, 308, 305, 306, 308, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.935427866Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.937061087Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.938339481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.940293406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00C.copy\" to \"/var/tmp/test_repair/8820/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.941005091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.941047955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00C.replace\" to \"/var/tmp/test_repair/8820/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.941190384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.942678717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.943188883Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.945211197Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.945727612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.945797352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.945935922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.947469638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.948036845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.950151856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.950657552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.950696027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.950847245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.952332677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.952845083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.954487013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/004.copy\" to \"/var/tmp/test_repair/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.955005347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/004.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.955067378Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/004.replace\" to \"/var/tmp/test_repair/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.955178402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.956723387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.957283115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.958966709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00D.copy\" to \"/var/tmp/test_repair/8820/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.959615894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.9596469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00D.replace\" to \"/var/tmp/test_repair/8820/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.959776031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:51.473628561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:51.473744954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:51.473790917Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 12 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:51.473836181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:51.473882374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.01466316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.014715803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.014782333Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: d62ec780-2fa6-448e-96ab-533d8bfa2533, session_id: 59a37b07-a36a-4c2b-84f6-5bbe2e9e940f, gen: 195 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.014812549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: d62ec780-2fa6-448e-96ab-533d8bfa2533, session_id: 59a37b07-a36a-4c2b-84f6-5bbe2e9e940f, gen: 195 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.014841764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.144453749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:51208"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.145207489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"13f59aa2-3eea-4b78-b154-b473064b2bba","remote_addr":"127.0.0.1:51208","local_addr":"0.0.0.0:12820","task":"repair","latency_us":134,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:44767","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178083019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:44767): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178186624Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"5"} {"msg":"Connection request from e5526f34-be2b-431e-b652-455c9beb2572 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.17876383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e5526f34-be2b-431e-b652-455c9beb2572, session_id: 34822fd7-3b35-423b-b220-c251c8e7d305, gen: 196 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178805663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: e5526f34-be2b-431e-b652-455c9beb2572, session_id: 34822fd7-3b35-423b-b220-c251c8e7d305, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179250299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [309, 309, 309, 302, 311, 308, 308, 305, 311, 308, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180135949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.199380959Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.199418694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.19944567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e5526f34-be2b-431e-b652-455c9beb2572, session_id: 34822fd7-3b35-423b-b220-c251c8e7d305, gen: 196 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.199468336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: e5526f34-be2b-431e-b652-455c9beb2572, session_id: 34822fd7-3b35-423b-b220-c251c8e7d305, gen: 196 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.19951433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.327527218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:63433"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.328284547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"01a4925b-c87f-43fb-8355-96da2510ceb6","remote_addr":"127.0.0.1:63433","local_addr":"0.0.0.0:12820","task":"repair","latency_us":162,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:61773","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.363575014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:61773): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.36367833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"6"} {"msg":"Connection request from 2e497477-54cd-4e95-b87a-812894da5943 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.364103587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 2e497477-54cd-4e95-b87a-812894da5943, session_id: 251f4ee7-5552-4382-bde1-3fb1660daad2, gen: 197 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.364134323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: 2e497477-54cd-4e95-b87a-812894da5943, session_id: 251f4ee7-5552-4382-bde1-3fb1660daad2, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.364580208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [309, 309, 309, 314, 311, 308, 308, 305, 314, 314, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.365518891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.367291222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.368611029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.370512692Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.37107153Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.371154438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.371286879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.372945216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.373461691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.375010105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/006.copy\" to \"/var/tmp/test_repair/8820/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.37551943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/006.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.375552066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/006.replace\" to \"/var/tmp/test_repair/8820/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.375761585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.377358282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.377857909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.379640059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.380225443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.380267007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.380415495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.382037819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.382568951Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.38435515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.384995167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.38504187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.385235962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.389165048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.389687432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.391541391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.392082412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.392118907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.392266255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.906308928Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.906380827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.906407023Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.906433089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.906466095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.447168952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:59.447205906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.44725451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 2e497477-54cd-4e95-b87a-812894da5943, session_id: 251f4ee7-5552-4382-bde1-3fb1660daad2, gen: 197 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.447275916Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 2e497477-54cd-4e95-b87a-812894da5943, session_id: 251f4ee7-5552-4382-bde1-3fb1660daad2, gen: 197 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.447298173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.57634591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:39743"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.5770998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"8b999d12-788c-4fb7-8c5a-d5fba8cb687d","remote_addr":"127.0.0.1:39743","local_addr":"0.0.0.0:12820","task":"repair","latency_us":135,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:53057","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.6092915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:53057): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609352791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"7"} {"msg":"Connection request from a0b5812b-2505-4f62-9e1f-f351dd0d2c28 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609755672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: a0b5812b-2505-4f62-9e1f-f351dd0d2c28, session_id: 72c66295-568b-4b15-8615-71a1a525d3db, gen: 198 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609794816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: a0b5812b-2505-4f62-9e1f-f351dd0d2c28, session_id: 72c66295-568b-4b15-8615-71a1a525d3db, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610201487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [309, 316, 316, 314, 311, 308, 316, 305, 316, 314, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611150768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.631970459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"7"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:59.632011182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"7"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.632039618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: a0b5812b-2505-4f62-9e1f-f351dd0d2c28, session_id: 72c66295-568b-4b15-8615-71a1a525d3db, gen: 198 } (ConnectionId(7)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.632072693Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: a0b5812b-2505-4f62-9e1f-f351dd0d2c28, session_id: 72c66295-568b-4b15-8615-71a1a525d3db, gen: 198 } (ConnectionId(7)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.632102699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.762432739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:42110"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.763136486Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"uri":"/region-info","method":"GET","req_id":"6ebbcb47-53fe-46ea-80ee-24e673b66225","remote_addr":"127.0.0.1:42110","local_addr":"0.0.0.0:12820","task":"repair","latency_us":137,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:63012","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.796172292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"main"} {"msg":"connection (127.0.0.1:63012): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.796217286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"id":"8"} {"msg":"Connection request from 744ec465-dd3b-4d6c-9d2f-3f3d52e5b2d4 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.79653148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 744ec465-dd3b-4d6c-9d2f-3f3d52e5b2d4, session_id: 08670651-9c44-4fa6-ad58-7cc2b7d8a3bd, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.796578953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"UpstairsConnection { upstairs_id: 744ec465-dd3b-4d6c-9d2f-3f3d52e5b2d4, session_id: 08670651-9c44-4fa6-ad58-7cc2b7d8a3bd, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.796966366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Current flush_numbers [0..12]: [322, 322, 319, 314, 311, 308, 316, 305, 316, 314, 319, 319]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.797886582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.799590023Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.800989669Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.803013853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.803628554Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.803673997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.803811377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.805280212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.805764572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.807636558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/009.copy\" to \"/var/tmp/test_repair/8820/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.808231462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/009.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.808269436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/009.replace\" to \"/var/tmp/test_repair/8820/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.808411585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.810105938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.810615463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.812255184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/003.copy\" to \"/var/tmp/test_repair/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.8127652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/003.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.812807963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/003.replace\" to \"/var/tmp/test_repair/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.812933585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.814598132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.815178987Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.816911584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.817453435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.817518195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.817657155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.819141838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.819697627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.821403838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/015.copy\" to \"/var/tmp/test_repair/8820/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.824443834Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/015.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.82447147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/015.replace\" to \"/var/tmp/test_repair/8820/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.824601161Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.826167233Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.826700445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.828464867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.829007998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.829036604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.829168094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.830632251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.831205787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.83282874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.833300451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.833350124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.833465457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.835098219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.835587377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.837247315Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.837834839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.837865395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.837994346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.839809281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.840306448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.842107705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/005.copy\" to \"/var/tmp/test_repair/8820/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.842614181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/005.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.842646366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/005.replace\" to \"/var/tmp/test_repair/8820/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.842770428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.844207309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.844718374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.851197738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.851671079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.851698405Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.851826246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.365836673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.365907313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.36592903Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 5 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.365952507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 9 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.365972284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.365991791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.366011288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.366030815Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.366050232Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:00.366069599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:38:03.907326456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"8"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:38:03.907382108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949,"task":"recv","id":"8"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:38:03.907407414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 744ec465-dd3b-4d6c-9d2f-3f3d52e5b2d4, session_id: 08670651-9c44-4fa6-ad58-7cc2b7d8a3bd, gen: 199 } (ConnectionId(8)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:38:03.907432021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949} {"msg":"upstairs UpstairsConnection { upstairs_id: 744ec465-dd3b-4d6c-9d2f-3f3d52e5b2d4, session_id: 08670651-9c44-4fa6-ad58-7cc2b7d8a3bd, gen: 199 } (ConnectionId(8)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:38:03.907456667Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1949}