{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.014875167Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.015493165Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.01551998Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.017533794Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.017555989Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 47aa4c863f974975b86a01cc6893b0f87d94aae1\nCommit timestamp: 2025-02-25T16:27:18.000000000Z branch: renovate/thiserror-1.x-lockfile\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.061567614Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.061622785Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.062613324Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.062944447Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.063119021Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.063151087Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.063226741Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"accepted connection from 127.0.0.1:38294","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.064168211Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:38294): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.064380731Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"0"} {"msg":"Connection request from 775f37a7-94e4-44df-ae3b-3ac2698c7a1e with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.064629799Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 775f37a7-94e4-44df-ae3b-3ac2698c7a1e, session_id: 0a822382-bc01-4aaf-a921-5f81564fac7c, gen: 191 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.064652843Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 775f37a7-94e4-44df-ae3b-3ac2698c7a1e, session_id: 0a822382-bc01-4aaf-a921-5f81564fac7c, gen: 191 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.064989917Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [319, 318, 323, 318, 326, 320, 320, 323, 326, 326, 318, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.065867045Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.067230825Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.069099131Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.070860767Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/019.copy\" to \"/var/tmp/test_repair/8820/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.071438968Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/019.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.071511222Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/019.replace\" to \"/var/tmp/test_repair/8820/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.071635365Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.073135731Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.073853498Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.075336301Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.075890577Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.075924773Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:21.076074492Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:21.590134247Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:21.590181135Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.133412648Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.133516108Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.133572239Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 775f37a7-94e4-44df-ae3b-3ac2698c7a1e, session_id: 0a822382-bc01-4aaf-a921-5f81564fac7c, gen: 191 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.133624798Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 775f37a7-94e4-44df-ae3b-3ac2698c7a1e, session_id: 0a822382-bc01-4aaf-a921-5f81564fac7c, gen: 191 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.133677519Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.26258722Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:45993"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.263367409Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"uri":"/region-info","method":"GET","req_id":"a96dc4ba-a0bb-4768-946c-555ae2de9eeb","remote_addr":"127.0.0.1:45993","local_addr":"0.0.0.0:12820","task":"repair","latency_us":215,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:55944","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.266148749Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:55944): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.266226684Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"1"} {"msg":"Connection request from c67ee2a8-d8c9-46fe-9b40-ced93b25d218 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.266578111Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c67ee2a8-d8c9-46fe-9b40-ced93b25d218, session_id: 9706f2d7-82ad-4655-8d9c-73b0bf183bc5, gen: 192 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.266606877Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: c67ee2a8-d8c9-46fe-9b40-ced93b25d218, session_id: 9706f2d7-82ad-4655-8d9c-73b0bf183bc5, gen: 192 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.267004453Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [319, 318, 323, 318, 326, 320, 320, 323, 326, 326, 318, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.267782391Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.28926841Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.289316799Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.289339794Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c67ee2a8-d8c9-46fe-9b40-ced93b25d218, session_id: 9706f2d7-82ad-4655-8d9c-73b0bf183bc5, gen: 192 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.289363428Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c67ee2a8-d8c9-46fe-9b40-ced93b25d218, session_id: 9706f2d7-82ad-4655-8d9c-73b0bf183bc5, gen: 192 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.289386482Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.424910526Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:52422"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.425673191Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"uri":"/region-info","method":"GET","req_id":"bbaba765-f4ee-48ae-9374-915f476e6888","remote_addr":"127.0.0.1:52422","local_addr":"0.0.0.0:12820","task":"repair","latency_us":141,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:38501","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.428525985Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:38501): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.42865557Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"2"} {"msg":"Connection request from 3e807f32-3507-4376-aaaa-2d4240202af8 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.428925042Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3e807f32-3507-4376-aaaa-2d4240202af8, session_id: 92e2410a-b553-49c2-a48b-bd8531a04a3c, gen: 193 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.428946646Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 3e807f32-3507-4376-aaaa-2d4240202af8, session_id: 92e2410a-b553-49c2-a48b-bd8531a04a3c, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.429381729Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [319, 318, 323, 318, 326, 320, 320, 323, 326, 326, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.430262237Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.431685818Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.432960161Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.434917425Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.435501616Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.435536793Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.435666087Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.436978478Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.437507959Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.439249931Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/017.copy\" to \"/var/tmp/test_repair/8820/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.439974259Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/017.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.440044723Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/017.replace\" to \"/var/tmp/test_repair/8820/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.440159425Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.441536167Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.442010238Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.443779925Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.444369678Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.444400183Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.444554453Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.445949929Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.446621787Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.448185976Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.448721288Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.448746193Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.448927587Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.45025424Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.450795483Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.452223106Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.452729652Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.452779282Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:25.452912678Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.967204686Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.967238392Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.967261107Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.967288132Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:25.967309266Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.505860846Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:29.505922378Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.505946643Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3e807f32-3507-4376-aaaa-2d4240202af8, session_id: 92e2410a-b553-49c2-a48b-bd8531a04a3c, gen: 193 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.505970427Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3e807f32-3507-4376-aaaa-2d4240202af8, session_id: 92e2410a-b553-49c2-a48b-bd8531a04a3c, gen: 193 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.505994512Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection from 127.0.0.1:45482","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.60982868Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:45482): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.609927789Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"3"} {"msg":"Connection request from a3b611aa-a1e8-4ebc-8090-17eb84c92c8f with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.610243949Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a3b611aa-a1e8-4ebc-8090-17eb84c92c8f, session_id: 2cded3b5-33a5-489c-90fc-00828ebed9ed, gen: 194 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.610270694Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: a3b611aa-a1e8-4ebc-8090-17eb84c92c8f, session_id: 2cded3b5-33a5-489c-90fc-00828ebed9ed, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.610672601Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [319, 318, 323, 318, 326, 320, 320, 323, 338, 326, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.611590946Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.632561507Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:29.632598004Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.632623599Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a3b611aa-a1e8-4ebc-8090-17eb84c92c8f, session_id: 2cded3b5-33a5-489c-90fc-00828ebed9ed, gen: 194 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.632649223Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a3b611aa-a1e8-4ebc-8090-17eb84c92c8f, session_id: 2cded3b5-33a5-489c-90fc-00828ebed9ed, gen: 194 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.632673698Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection from 127.0.0.1:51149","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.73359471Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:51149): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.733635058Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"4"} {"msg":"Connection request from 8fac43ac-cc77-49ad-9fd6-805babb18728 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.734003928Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 8fac43ac-cc77-49ad-9fd6-805babb18728, session_id: e0dcc4cd-0434-46e0-9928-4c121a3ab924, gen: 195 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.734039845Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 8fac43ac-cc77-49ad-9fd6-805babb18728, session_id: e0dcc4cd-0434-46e0-9928-4c121a3ab924, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.734611394Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [339, 339, 323, 318, 326, 320, 339, 339, 338, 340, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.735508315Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.739286866Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.73998687Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.741919408Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.742501599Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.742535006Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.742711849Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.745032832Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.745564824Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.74711811Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.747669185Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.74769485Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.747823075Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.749138306Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.749615477Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.751290526Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.751823508Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.751849112Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.752018265Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.75341026Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.753967287Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.755499789Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/019.copy\" to \"/var/tmp/test_repair/8820/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.756033981Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/019.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.756069057Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/019.replace\" to \"/var/tmp/test_repair/8820/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.756195142Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.757551731Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.758040304Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.759525677Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00C.copy\" to \"/var/tmp/test_repair/8820/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.76006327Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.760111269Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00C.replace\" to \"/var/tmp/test_repair/8820/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:29.760323369Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:30.278026678Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 12 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:30.27808629Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:30.278112255Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:30.278135239Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:30.278160874Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.818781068Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:33.818821616Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.81889586Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 8fac43ac-cc77-49ad-9fd6-805babb18728, session_id: e0dcc4cd-0434-46e0-9928-4c121a3ab924, gen: 195 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.818928126Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 8fac43ac-cc77-49ad-9fd6-805babb18728, session_id: e0dcc4cd-0434-46e0-9928-4c121a3ab924, gen: 195 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.818956782Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection from 127.0.0.1:39363","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.921929546Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:39363): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.921978255Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"5"} {"msg":"Connection request from c0a2e298-6df5-4e14-84d4-956b7b1b600c with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.922509806Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c0a2e298-6df5-4e14-84d4-956b7b1b600c, session_id: 4d576534-1b1f-42e8-9f86-8d18bc78f904, gen: 196 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.922548364Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: c0a2e298-6df5-4e14-84d4-956b7b1b600c, session_id: 4d576534-1b1f-42e8-9f86-8d18bc78f904, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.922981366Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [339, 339, 342, 318, 326, 320, 339, 339, 338, 340, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.923886579Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.944979963Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:33.945015759Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.945044905Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c0a2e298-6df5-4e14-84d4-956b7b1b600c, session_id: 4d576534-1b1f-42e8-9f86-8d18bc78f904, gen: 196 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.94507021Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c0a2e298-6df5-4e14-84d4-956b7b1b600c, session_id: 4d576534-1b1f-42e8-9f86-8d18bc78f904, gen: 196 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:33.945091224Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection from 127.0.0.1:53498","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.044717399Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:53498): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.04477613Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"6"} {"msg":"Connection request from 57224a03-66e3-4b2b-9629-608d5e61f33e with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.045082069Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 57224a03-66e3-4b2b-9629-608d5e61f33e, session_id: ec94cf85-0aac-423c-b398-3b253c90f9a8, gen: 197 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.045107684Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 57224a03-66e3-4b2b-9629-608d5e61f33e, session_id: ec94cf85-0aac-423c-b398-3b253c90f9a8, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.045555009Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [339, 345, 343, 343, 326, 320, 343, 339, 338, 340, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.046413273Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.047886194Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.048596339Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.050469716Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.051074452Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.051110949Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.051264098Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.053751633Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.054455177Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.056072415Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/004.copy\" to \"/var/tmp/test_repair/8820/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.056604177Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/004.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.056638343Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/004.replace\" to \"/var/tmp/test_repair/8820/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:34.056745344Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 4 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:34.571174329Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:34.571217627Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.11266552Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.112722941Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.112743515Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 57224a03-66e3-4b2b-9629-608d5e61f33e, session_id: ec94cf85-0aac-423c-b398-3b253c90f9a8, gen: 197 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.112765019Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 57224a03-66e3-4b2b-9629-608d5e61f33e, session_id: ec94cf85-0aac-423c-b398-3b253c90f9a8, gen: 197 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.112785853Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.240634352Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:49716"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.241327714Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"uri":"/region-info","method":"GET","req_id":"655b87eb-b5e9-4b62-826b-4b2374b8cf58","remote_addr":"127.0.0.1:49716","local_addr":"0.0.0.0:12820","task":"repair","latency_us":135,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:54681","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.243941383Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:54681): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.24403252Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"7"} {"msg":"Connection request from d391cf40-ecbd-47c8-9731-173708c1720d with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.24450589Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: d391cf40-ecbd-47c8-9731-173708c1720d, session_id: 37f51c01-6491-407d-ad20-332922bfdfd8, gen: 198 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.244524074Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: d391cf40-ecbd-47c8-9731-173708c1720d, session_id: 37f51c01-6491-407d-ad20-332922bfdfd8, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.245040803Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [339, 345, 343, 343, 346, 320, 343, 339, 338, 340, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.245746337Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.26678438Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"7"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.266816766Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"7"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.26683807Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: d391cf40-ecbd-47c8-9731-173708c1720d, session_id: 37f51c01-6491-407d-ad20-332922bfdfd8, gen: 198 } (ConnectionId(7)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.266860995Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: d391cf40-ecbd-47c8-9731-173708c1720d, session_id: 37f51c01-6491-407d-ad20-332922bfdfd8, gen: 198 } (ConnectionId(7)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.266883589Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.400656489Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:53113"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.401383927Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"uri":"/region-info","method":"GET","req_id":"0fff66ab-092a-4649-9b19-ce2c1b3408b8","remote_addr":"127.0.0.1:53113","local_addr":"0.0.0.0:12820","task":"repair","latency_us":143,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:37722","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.404168468Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:37722): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.404324738Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"id":"8"} {"msg":"Connection request from af20c823-f2f6-411e-8b16-1c72986893b7 with version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.404602741Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: af20c823-f2f6-411e-8b16-1c72986893b7, session_id: ec3c8e68-d6d0-4bf5-bc3b-de842a1f6177, gen: 199 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.404628796Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"UpstairsConnection { upstairs_id: af20c823-f2f6-411e-8b16-1c72986893b7, session_id: ec3c8e68-d6d0-4bf5-bc3b-de842a1f6177, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.405009979Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Current flush_numbers [0..12]: [339, 345, 343, 343, 346, 320, 343, 339, 338, 340, 334, 329]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.405848079Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.407307437Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.408536692Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.410610467Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00D.copy\" to \"/var/tmp/test_repair/8820/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.411272444Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.41130707Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00D.replace\" to \"/var/tmp/test_repair/8820/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.411459769Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.412762668Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.413258642Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.414904616Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.415413133Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.415443729Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.415583076Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.41723039Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.417708431Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.419322119Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.419846839Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.419919533Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.420078183Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.421862934Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.422506207Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.424172924Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.424704176Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.424741243Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.424877109Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01B.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.426170416Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:27 Found repair files: [\"01B\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.426720961Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 27 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.428320976Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01B.copy\" to \"/var/tmp/test_repair/8820/00/000/01B.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.428896286Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.428959148Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01B.replace\" to \"/var/tmp/test_repair/8820/00/000/01B.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.429153015Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.4305977Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.431112859Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.432617295Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/016.copy\" to \"/var/tmp/test_repair/8820/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.433249086Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/016.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.433279242Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/016.replace\" to \"/var/tmp/test_repair/8820/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.433449834Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.434756834Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.435242166Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.436944121Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.437434414Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.437489405Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.43762166Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/017.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.439180007Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:23 Found repair files: [\"017\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.439858927Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 23 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.441180879Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/017.copy\" to \"/var/tmp/test_repair/8820/00/000/017.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.441697408Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/017.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.441729854Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/017.replace\" to \"/var/tmp/test_repair/8820/00/000/017.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.441889414Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.443459754Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.444123881Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.450492145Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/003.copy\" to \"/var/tmp/test_repair/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.450971927Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/003.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.451000242Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/003.replace\" to \"/var/tmp/test_repair/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.451118315Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.452490477Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.453018697Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.454846916Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.455401422Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.455431067Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.45554985Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/013.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.45701642Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:19 Found repair files: [\"013\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.457521436Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 19 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.45906067Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/013.copy\" to \"/var/tmp/test_repair/8820/00/000/013.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.459702452Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/013.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.459738229Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/013.replace\" to \"/var/tmp/test_repair/8820/00/000/013.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.459841529Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.461138386Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.461685871Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.463208131Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.463740463Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.463765408Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.463954244Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.465412582Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.465967017Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.467640677Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/015.copy\" to \"/var/tmp/test_repair/8820/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.468146513Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/015.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.468174378Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/015.replace\" to \"/var/tmp/test_repair/8820/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:38.468299713Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983296286Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983393424Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983453776Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983490213Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983523819Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 19 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983564227Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983593993Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983621518Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983645942Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 23 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983669307Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983694222Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 27 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983721097Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:38.983746652Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:42.526304317Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"8"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-02-25T17:26:42.52637089Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950,"task":"recv","id":"8"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:42.526401475Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: af20c823-f2f6-411e-8b16-1c72986893b7, session_id: ec3c8e68-d6d0-4bf5-bc3b-de842a1f6177, gen: 199 } (ConnectionId(8)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:42.526422149Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: af20c823-f2f6-411e-8b16-1c72986893b7, session_id: ec3c8e68-d6d0-4bf5-bc3b-de842a1f6177, gen: 199 } (ConnectionId(8)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-02-25T17:26:42.526443023Z","hostname":"w-01JMZ1RC1ATKB5GDJGK1D8DCQ0","pid":1950}