{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.024492666Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.024985504Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.025009809Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.028207938Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.028230571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: abd5430f1508af0bc008143ea2566277ef714ec2\nCommit timestamp: 2025-01-17T20:55:03.000000000Z branch: alan/tokio-to-140\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.071592835Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.071649248Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.072639777Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.07302145Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.073195532Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.07322683Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.073249553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"accepted connection from 127.0.0.1:63871","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.073870056Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:63871): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.074043668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"0"} {"msg":"Connection request from c5dff4e4-cd68-48cb-acfb-f150239e794b with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.074350077Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c5dff4e4-cd68-48cb-acfb-f150239e794b, session_id: c3818b20-c0ae-4bc8-a270-4d4f197359d2, gen: 191 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.074380935Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: c5dff4e4-cd68-48cb-acfb-f150239e794b, session_id: c3818b20-c0ae-4bc8-a270-4d4f197359d2, gen: 191 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.074823193Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [328, 329, 338, 340, 340, 339, 339, 339, 334, 327, 331, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.075626973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/018.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.077124728Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:24 Found repair files: [\"018\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.078936937Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 24 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.080683358Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/018.copy\" to \"/var/tmp/test_repair/8820/00/000/018.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.081228057Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/018.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.081296147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/018.replace\" to \"/var/tmp/test_repair/8820/00/000/018.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.081423541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.082777032Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:27 Found repair files: [\"01B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.083451486Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 27 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.085192974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01B.copy\" to \"/var/tmp/test_repair/8820/00/000/01B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.085723394Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.085756443Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01B.replace\" to \"/var/tmp/test_repair/8820/00/000/01B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.085880686Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.087212444Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.087743455Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.089519963Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.090193547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.090222884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.090411684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.0916873Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.092215068Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.094125895Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.096794945Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.096822431Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.096949625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.098399833Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.098914094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.100746625Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.101264978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.101292894Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.101417367Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.102675962Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.103254401Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.104932821Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.105564791Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.105589855Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.105758704Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.10725644Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.107757902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.109391568Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.109922858Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.109956628Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:20.110070254Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.624824914Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.62487047Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.624888571Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.62490349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 24 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.624917298Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 27 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.624931196Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:20.624944844Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.162332634Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:24.162476859Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.162526297Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c5dff4e4-cd68-48cb-acfb-f150239e794b, session_id: c3818b20-c0ae-4bc8-a270-4d4f197359d2, gen: 191 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.162574926Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c5dff4e4-cd68-48cb-acfb-f150239e794b, session_id: c3818b20-c0ae-4bc8-a270-4d4f197359d2, gen: 191 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.162600411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.294384905Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:48452"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.295150733Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"uri":"/region-info","method":"GET","req_id":"0c095ce8-d1f4-4eab-90cc-c748a063a8c4","remote_addr":"127.0.0.1:48452","local_addr":"0.0.0.0:12820","task":"repair","latency_us":198,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:56521","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.297962886Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:56521): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298017388Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"1"} {"msg":"Connection request from 6f56a947-0808-49d8-8575-8b9456f36dd7 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298270076Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 6f56a947-0808-49d8-8575-8b9456f36dd7, session_id: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba, gen: 192 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298300063Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: 6f56a947-0808-49d8-8575-8b9456f36dd7, session_id: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba, gen: 192 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.298778493Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [328, 329, 338, 340, 340, 339, 339, 341, 334, 327, 331, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.299733441Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.321968477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:24.32200845Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.322037347Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 6f56a947-0808-49d8-8575-8b9456f36dd7, session_id: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba, gen: 192 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.322063002Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 6f56a947-0808-49d8-8575-8b9456f36dd7, session_id: 467ea12b-3d3a-4e7d-bb16-616ce8c752ba, gen: 192 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.322086156Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.454798982Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:65251"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.455516372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"uri":"/region-info","method":"GET","req_id":"22a0b93a-5d06-4210-aa7a-fba3b9269809","remote_addr":"127.0.0.1:65251","local_addr":"0.0.0.0:12820","task":"repair","latency_us":135,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:38894","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.458265979Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:38894): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.458295296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"2"} {"msg":"Connection request from d7d5b624-adc6-4505-8ca4-73bdc992f364 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.458690547Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: d7d5b624-adc6-4505-8ca4-73bdc992f364, session_id: 7e145bd6-ff0f-4e34-a89f-02d18424344f, gen: 193 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.458716082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: d7d5b624-adc6-4505-8ca4-73bdc992f364, session_id: 7e145bd6-ff0f-4e34-a89f-02d18424344f, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.459153888Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [328, 329, 338, 340, 340, 339, 343, 341, 334, 327, 331, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.460139554Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.461656721Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.463074839Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.465133643Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.465717614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.465780081Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.465896749Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.467439281Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.468108752Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.469828257Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.470391266Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.470427147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.470549028Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.471862766Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.472367341Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.474186514Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.474659561Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.474701635Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.474850602Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.476187734Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.476716553Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.478625338Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.479108381Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.479149405Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.479286875Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.48063044Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.481353353Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.484836278Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/017.copy\" to \"/var/tmp/test_repair/8820/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.485342374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/017.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.485373793Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/017.replace\" to \"/var/tmp/test_repair/8820/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.485502218Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.487035794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.487711459Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.489281427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.489812637Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.489841094Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.489994884Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.491363513Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.491868569Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.493527798Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.494030652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.494075349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.494256404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.495827913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.496500816Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.49813306Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/004.copy\" to \"/var/tmp/test_repair/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.498764309Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/004.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.498787623Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/004.replace\" to \"/var/tmp/test_repair/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.498923452Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.500212856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.500731449Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.50242696Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.502926762Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.502963323Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.503147301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.504644976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.505167372Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.509308302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/006.copy\" to \"/var/tmp/test_repair/8820/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.509831678Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/006.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.509862866Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/006.replace\" to \"/var/tmp/test_repair/8820/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:24.509978713Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024116552Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 6 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024151712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.02416427Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024182901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024201312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024218542Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024236302Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024253822Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024270862Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:25.024288222Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.563837147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:28.563918605Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.564151171Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: d7d5b624-adc6-4505-8ca4-73bdc992f364, session_id: 7e145bd6-ff0f-4e34-a89f-02d18424344f, gen: 193 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.564210986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: d7d5b624-adc6-4505-8ca4-73bdc992f364, session_id: 7e145bd6-ff0f-4e34-a89f-02d18424344f, gen: 193 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.564268949Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection from 127.0.0.1:57280","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.668921186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:57280): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669016442Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"3"} {"msg":"Connection request from dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669420668Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e, session_id: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e, gen: 194 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669447944Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e, session_id: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.669910974Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [328, 329, 338, 340, 344, 339, 344, 344, 344, 327, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.670731404Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.691851928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:28.691886488Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.691907491Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e, session_id: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e, gen: 194 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.691929794Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: dbfa8cb4-8abb-4424-a49b-84c2cd2f4c2e, session_id: b01e58e8-dced-4d7c-b3e2-4636e82c1a8e, gen: 194 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.691952437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection from 127.0.0.1:54017","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.79574301Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:54017): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.795878789Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"4"} {"msg":"Connection request from bc54ddf2-7aff-4bc0-b556-383df88ffc3e with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.796258691Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: bc54ddf2-7aff-4bc0-b556-383df88ffc3e, session_id: fc189005-1add-444c-a64c-464ec791930b, gen: 195 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.796294872Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: bc54ddf2-7aff-4bc0-b556-383df88ffc3e, session_id: fc189005-1add-444c-a64c-464ec791930b, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.796754541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [328, 348, 346, 347, 344, 339, 344, 344, 344, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.797615204Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.799049112Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.799751322Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.801644939Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.802229411Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.802299782Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.802473383Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.804762931Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.80525723Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.806810718Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.807396121Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.807422986Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.807548009Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.809151667Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.809631927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.811304975Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/004.copy\" to \"/var/tmp/test_repair/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.815096981Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/004.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.815171135Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/004.replace\" to \"/var/tmp/test_repair/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.81538453Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.818373877Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.818865284Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.82070676Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.82120197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.821229286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.821351847Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.822805707Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.823329983Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.82482919Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.825338197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.825384814Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.825519523Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.827082597Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.82772194Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.829240158Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.829765295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.82979203Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.829905606Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.831506913Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.832023434Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.833514856Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.834033629Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.834063917Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:28.83418927Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348853787Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.34891047Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348929681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348947312Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348964592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348982082Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:29.348999132Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.891813256Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:32.891874712Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.891898165Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: bc54ddf2-7aff-4bc0-b556-383df88ffc3e, session_id: fc189005-1add-444c-a64c-464ec791930b, gen: 195 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.891920398Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: bc54ddf2-7aff-4bc0-b556-383df88ffc3e, session_id: fc189005-1add-444c-a64c-464ec791930b, gen: 195 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.891943902Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection from 127.0.0.1:59513","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99593169Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:59513): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.995989684Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"5"} {"msg":"Connection request from 0027a692-d575-45e8-a4d9-6dd89ffae800 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.996385365Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 0027a692-d575-45e8-a4d9-6dd89ffae800, session_id: 8e301242-2608-4c18-8a03-a555931cf0ba, gen: 196 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.996422507Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: 0027a692-d575-45e8-a4d9-6dd89ffae800, session_id: 8e301242-2608-4c18-8a03-a555931cf0ba, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.99685596Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [349, 348, 346, 347, 349, 339, 344, 349, 349, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:32.997603557Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.019558759Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.019596031Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.019622277Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 0027a692-d575-45e8-a4d9-6dd89ffae800, session_id: 8e301242-2608-4c18-8a03-a555931cf0ba, gen: 196 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.01964508Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 0027a692-d575-45e8-a4d9-6dd89ffae800, session_id: 8e301242-2608-4c18-8a03-a555931cf0ba, gen: 196 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.019668374Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection from 127.0.0.1:60026","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.121627376Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:60026): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.121664197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"6"} {"msg":"Connection request from 5df0043b-e140-40e1-826d-2beef3d2c0a2 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.121978841Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.122038326Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.122480474Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [349, 348, 346, 347, 352, 350, 350, 350, 352, 345, 331, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.123344059Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.124860115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.125499869Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.127306745Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.127841328Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.127910498Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.128096437Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.130704041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.131259115Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.132937286Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/016.copy\" to \"/var/tmp/test_repair/8820/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133455829Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/016.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133494702Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/016.replace\" to \"/var/tmp/test_repair/8820/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.133682212Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.135192054Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.135722464Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137355699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137920349Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.137954108Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.138102295Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.139381993Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.139887458Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.141493757Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.142002475Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.142030711Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.142171103Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.143862041Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.146464582Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.14837716Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00A.copy\" to \"/var/tmp/test_repair/8820/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148909772Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.148945282Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00A.replace\" to \"/var/tmp/test_repair/8820/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.149050414Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.150345891Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.151053465Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.152890999Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.153454368Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.153505978Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:33.15362959Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669323699Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.66937675Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669404666Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669427509Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.66944587Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:33.669457647Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208482145Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.208529583Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208552916Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.20861147Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 5df0043b-e140-40e1-826d-2beef3d2c0a2, session_id: 52281691-8105-432b-b8bd-8faa371fb9cb, gen: 197 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.208641588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.338930928Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:57405"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.339664207Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"uri":"/region-info","method":"GET","req_id":"02545393-6300-43ce-84cd-ffc902843b39","remote_addr":"127.0.0.1:57405","local_addr":"0.0.0.0:12820","task":"repair","latency_us":138,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:46942","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342319079Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:46942): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342397795Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"7"} {"msg":"Connection request from c9f591c0-4528-4e75-9b2a-fad223b5379e with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.342789174Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.3428181Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.343290727Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [353, 353, 353, 347, 352, 350, 350, 350, 352, 345, 353, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.344201539Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366383614Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"7"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:37.366412781Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"7"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366434343Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } (ConnectionId(7)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366455976Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: c9f591c0-4528-4e75-9b2a-fad223b5379e, session_id: d5170d1b-c9bb-4c15-80ad-fd31d55d99a0, gen: 198 } (ConnectionId(7)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.366477729Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.495446517Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:58127"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.496180016Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"uri":"/region-info","method":"GET","req_id":"5f34cfe2-f04a-4813-b8c9-ca8cc0a0240b","remote_addr":"127.0.0.1:58127","local_addr":"0.0.0.0:12820","task":"repair","latency_us":134,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:43137","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.498859422Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"main"} {"msg":"connection (127.0.0.1:43137): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.498951106Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"id":"8"} {"msg":"Connection request from 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1 with version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.49938709Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.499418049Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.499884961Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Current flush_numbers [0..12]: [354, 354, 353, 347, 352, 350, 354, 350, 352, 345, 354, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.500664177Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.502172929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.50347592Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.505352037Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.505943973Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.505971909Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.506105477Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.507679297Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.508330588Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510138714Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00B.copy\" to \"/var/tmp/test_repair/8820/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510663371Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510703155Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00B.replace\" to \"/var/tmp/test_repair/8820/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.510850681Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.512216299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.512743197Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.514441319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.514966476Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.515000716Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.515164262Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.51656409Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.517232671Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.519097701Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/017.copy\" to \"/var/tmp/test_repair/8820/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.521786482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/017.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.5218167Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/017.replace\" to \"/var/tmp/test_repair/8820/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.521943264Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00F.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.5232715Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:15 Found repair files: [\"00F\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.523791464Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 15 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.525479141Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00F.copy\" to \"/var/tmp/test_repair/8820/00/000/00F.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526016575Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00F.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526052296Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00F.replace\" to \"/var/tmp/test_repair/8820/00/000/00F.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.526136985Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.527720901Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.528401299Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530026048Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00A.copy\" to \"/var/tmp/test_repair/8820/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530569226Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530603536Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00A.replace\" to \"/var/tmp/test_repair/8820/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.530784541Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.532107014Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.53261275Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.533934933Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.53449538Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.534522436Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:37.534650521Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049681652Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.0497813Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 10 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049825216Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 11 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.04986631Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049890514Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 15 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.049912387Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:38.04993482Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.58798787Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"8"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-17T21:19:41.588026213Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944,"task":"recv","id":"8"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.588088319Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } (ConnectionId(8)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.588117186Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944} {"msg":"upstairs UpstairsConnection { upstairs_id: 78d544f2-c9d4-4c50-a32a-00b4dd1e87d1, session_id: de8b85d5-1e0b-4654-8192-de4da93cb3ad, gen: 199 } (ConnectionId(8)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-17T21:19:41.588138929Z","hostname":"w-01JHV1KHN49PGZ24K9A4XN75MW","pid":1944}