{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.205826854Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.206344763Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.206363301Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.215179225Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.215210312Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 664b05998b747a86e13a12bb6c5e4428458d8bd9\nCommit timestamp: 2024-10-31T20:43:19.000000000Z branch: alan/crutest-volume-info\nrustc: 1.80.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.263631453Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.263681807Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.265028787Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.265464465Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.265654373Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.26567809Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.265775879Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"accepted connection from 127.0.0.1:57147","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.292198627Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:57147): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.292442518Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"0"} {"msg":"Connection request from ff33b496-aa0d-4a88-b5f2-20f5c3fc13d1 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.292814734Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: ff33b496-aa0d-4a88-b5f2-20f5c3fc13d1, session_id: 9714e085-81d3-4cf4-86ab-37707da0e053, gen: 193 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.29284677Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: ff33b496-aa0d-4a88-b5f2-20f5c3fc13d1, session_id: 9714e085-81d3-4cf4-86ab-37707da0e053, gen: 193 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.29326946Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 335, 335, 321, 335, 336, 335, 329, 327, 337, 338, 338]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.294232566Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/015.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.296866174Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:21 Found repair files: [\"015\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.298400552Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 21 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.301379099Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/015.copy\" to \"/var/tmp/test_repair/8820/00/000/015.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.301985817Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/015.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.303625343Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/015.replace\" to \"/var/tmp/test_repair/8820/00/000/015.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.305646103Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.310151099Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.310897061Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.315159176Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.315718299Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.31698285Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.318335459Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.322169305Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.322907977Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.327201818Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.329640629Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.330763496Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.333045526Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/014.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.337603326Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:20 Found repair files: [\"014\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.338284475Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 20 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.341102331Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/014.copy\" to \"/var/tmp/test_repair/8820/00/000/014.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.341692781Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/014.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.343177005Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/014.replace\" to \"/var/tmp/test_repair/8820/00/000/014.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.344608815Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.349610312Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.35039377Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.353830342Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.354333373Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.355310217Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:50.357779794Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:50.793248711Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:50.793289006Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:50.793313933Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 20 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:50.79333558Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 21 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:50.793348639Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.412667657Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:54.412757076Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.412786743Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: ff33b496-aa0d-4a88-b5f2-20f5c3fc13d1, session_id: 9714e085-81d3-4cf4-86ab-37707da0e053, gen: 193 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.412811Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: ff33b496-aa0d-4a88-b5f2-20f5c3fc13d1, session_id: 9714e085-81d3-4cf4-86ab-37707da0e053, gen: 193 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.412841586Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection from 127.0.0.1:60452","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.546125599Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:60452): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.546225277Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"1"} {"msg":"Connection request from e7488be6-9184-46a8-9a90-a813477646e4 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.546678943Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: e7488be6-9184-46a8-9a90-a813477646e4, session_id: ea1cbc38-f308-465a-94a0-0205a1d79df8, gen: 194 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.546717249Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: e7488be6-9184-46a8-9a90-a813477646e4, session_id: ea1cbc38-f308-465a-94a0-0205a1d79df8, gen: 194 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.547202971Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 340, 340, 321, 335, 336, 335, 340, 327, 337, 338, 338]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.548290032Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.571953048Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"1"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:54.571989643Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"1"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.572012971Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: e7488be6-9184-46a8-9a90-a813477646e4, session_id: ea1cbc38-f308-465a-94a0-0205a1d79df8, gen: 194 } (ConnectionId(1)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.572033118Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: e7488be6-9184-46a8-9a90-a813477646e4, session_id: ea1cbc38-f308-465a-94a0-0205a1d79df8, gen: 194 } (ConnectionId(1)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.572054256Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection from 127.0.0.1:35564","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.704189925Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:35564): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.704248418Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"2"} {"msg":"Connection request from 3b0ff1e2-f42b-47d3-a3a7-7d80d35ad411 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.704610935Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3b0ff1e2-f42b-47d3-a3a7-7d80d35ad411, session_id: d590a5e8-3217-4710-ae49-df6aa9f113a9, gen: 195 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.704642891Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 3b0ff1e2-f42b-47d3-a3a7-7d80d35ad411, session_id: d590a5e8-3217-4710-ae49-df6aa9f113a9, gen: 195 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.705203974Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 340, 340, 321, 335, 341, 341, 340, 327, 337, 338, 341]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.706097169Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.708641977Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:27 Found repair files: [\"01B\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.709440632Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 27 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.712673879Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01B.copy\" to \"/var/tmp/test_repair/8820/00/000/01B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.713191848Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.714108239Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01B.replace\" to \"/var/tmp/test_repair/8820/00/000/01B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.716510185Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.720860889Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.721653325Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.725429667Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/016.copy\" to \"/var/tmp/test_repair/8820/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.725954505Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/016.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.726987473Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/016.replace\" to \"/var/tmp/test_repair/8820/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.728756783Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.733243161Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.733767629Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.737605664Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/007.copy\" to \"/var/tmp/test_repair/8820/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.73823135Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/007.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.739192546Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/007.replace\" to \"/var/tmp/test_repair/8820/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:54.767623726Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 7 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:55.20446706Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:55.204532452Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 27 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:55.204556349Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.835717183Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"2"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:58.835778466Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"2"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.835842038Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3b0ff1e2-f42b-47d3-a3a7-7d80d35ad411, session_id: d590a5e8-3217-4710-ae49-df6aa9f113a9, gen: 195 } (ConnectionId(2)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.835865765Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 3b0ff1e2-f42b-47d3-a3a7-7d80d35ad411, session_id: d590a5e8-3217-4710-ae49-df6aa9f113a9, gen: 195 } (ConnectionId(2)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.835886123Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.964715584Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:54880"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.965566423Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"uri":"/region-info","method":"GET","req_id":"918b846c-2be4-4d53-a2e5-6089d862e694","remote_addr":"127.0.0.1:54880","local_addr":"0.0.0.0:12820","task":"repair","latency_us":207,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:45316","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.998436687Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:45316): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.998605747Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"3"} {"msg":"Connection request from c6c1daf8-018a-4d6b-a3d2-3cc92ec3f291 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.998857957Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c6c1daf8-018a-4d6b-a3d2-3cc92ec3f291, session_id: 003b196f-64b9-4939-b95d-db6553ef45d2, gen: 196 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.998893493Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: c6c1daf8-018a-4d6b-a3d2-3cc92ec3f291, session_id: 003b196f-64b9-4939-b95d-db6553ef45d2, gen: 196 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:58.999323242Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 340, 340, 321, 335, 341, 341, 342, 327, 337, 338, 341]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.000365329Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.028411404Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"3"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.028462888Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"3"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.028488275Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c6c1daf8-018a-4d6b-a3d2-3cc92ec3f291, session_id: 003b196f-64b9-4939-b95d-db6553ef45d2, gen: 196 } (ConnectionId(3)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.028510043Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: c6c1daf8-018a-4d6b-a3d2-3cc92ec3f291, session_id: 003b196f-64b9-4939-b95d-db6553ef45d2, gen: 196 } (ConnectionId(3)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.02853157Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.157850793Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"local_addr":"0.0.0.0:12820","task":"repair","remote_addr":"127.0.0.1:33863"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.158602344Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"uri":"/region-info","method":"GET","req_id":"8f61f61e-7e58-4644-8f22-c94e7c9454bc","remote_addr":"127.0.0.1:33863","local_addr":"0.0.0.0:12820","task":"repair","latency_us":146,"response_code":"200"} {"msg":"accepted connection from 127.0.0.1:48456","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.191449351Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:48456): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.191519362Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"4"} {"msg":"Connection request from 7f82c102-7f4e-43e5-9cfa-a6d7348dfe7e with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.191988147Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 7f82c102-7f4e-43e5-9cfa-a6d7348dfe7e, session_id: 0f9ba9e1-1f4e-4d6d-b355-cb90b9051867, gen: 197 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.192030042Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 7f82c102-7f4e-43e5-9cfa-a6d7348dfe7e, session_id: 0f9ba9e1-1f4e-4d6d-b355-cb90b9051867, gen: 197 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.192508005Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 343, 343, 321, 335, 341, 341, 343, 343, 337, 338, 341]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.193511896Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.196241233Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.19753168Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.203908664Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/008.copy\" to \"/var/tmp/test_repair/8820/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.204438091Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/008.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.205614212Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/008.replace\" to \"/var/tmp/test_repair/8820/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.206917028Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.211906316Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.212609963Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.215782307Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.216328922Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.217232285Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.219624781Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.224319125Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.225018512Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.228656591Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00D.copy\" to \"/var/tmp/test_repair/8820/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.229274838Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.230573424Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00D.replace\" to \"/var/tmp/test_repair/8820/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.232020342Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01A.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.236052874Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:26 Found repair files: [\"01A\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.236564834Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 26 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.241059631Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01A.copy\" to \"/var/tmp/test_repair/8820/00/000/01A.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.241619804Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01A.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.242769628Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01A.replace\" to \"/var/tmp/test_repair/8820/00/000/01A.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.245093733Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.248900192Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.249412121Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.25363937Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/002.copy\" to \"/var/tmp/test_repair/8820/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.254236629Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/002.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.255463474Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/002.replace\" to \"/var/tmp/test_repair/8820/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:07:59.257693799Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.692398487Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.69245468Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 8 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.692475968Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.692494705Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 26 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:07:59.692513653Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.313999394Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"4"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:03.314065676Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"4"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.314095302Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 7f82c102-7f4e-43e5-9cfa-a6d7348dfe7e, session_id: 0f9ba9e1-1f4e-4d6d-b355-cb90b9051867, gen: 197 } (ConnectionId(4)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.314124819Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 7f82c102-7f4e-43e5-9cfa-a6d7348dfe7e, session_id: 0f9ba9e1-1f4e-4d6d-b355-cb90b9051867, gen: 197 } (ConnectionId(4)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.314153895Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection from 127.0.0.1:45394","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.448386196Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:45394): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.448429161Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"5"} {"msg":"Connection request from 1b1a2744-56e5-46e6-801f-e140e2d3e75b with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.4488605Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 1b1a2744-56e5-46e6-801f-e140e2d3e75b, session_id: e536ed20-eefa-4b3a-bf70-95ab2edf7a9d, gen: 198 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.448889696Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: 1b1a2744-56e5-46e6-801f-e140e2d3e75b, session_id: e536ed20-eefa-4b3a-bf70-95ab2edf7a9d, gen: 198 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.449457439Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 341]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.450360242Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.47695329Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"5"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:03.477003894Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"5"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.477031551Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 1b1a2744-56e5-46e6-801f-e140e2d3e75b, session_id: e536ed20-eefa-4b3a-bf70-95ab2edf7a9d, gen: 198 } (ConnectionId(5)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.477058068Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: 1b1a2744-56e5-46e6-801f-e140e2d3e75b, session_id: e536ed20-eefa-4b3a-bf70-95ab2edf7a9d, gen: 198 } (ConnectionId(5)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.477084815Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"accepted connection from 127.0.0.1:57015","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614387981Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"main"} {"msg":"connection (127.0.0.1:57015): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614497628Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"id":"6"} {"msg":"Connection request from a4789ed1-39f5-432e-8d86-819e12466435 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614890401Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614926167Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615364675Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Current flush_numbers [0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 346]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616410091Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.619689513Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.620382591Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.623577452Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/003.copy\" to \"/var/tmp/test_repair/8820/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.624113148Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/003.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.624902025Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/003.replace\" to \"/var/tmp/test_repair/8820/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.62840364Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.633664026Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.634401479Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.638834364Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.639360151Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.640579547Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.643302984Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.649645912Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.650374806Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.653906737Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.654515665Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.655923108Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.657749022Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.662852707Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.663407811Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.667424845Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/011.copy\" to \"/var/tmp/test_repair/8820/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.667990058Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/011.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.66898508Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/011.replace\" to \"/var/tmp/test_repair/8820/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.670475703Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.675891861Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.676605617Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.679843963Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/010.copy\" to \"/var/tmp/test_repair/8820/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.680433973Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/010.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.681341115Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/010.replace\" to \"/var/tmp/test_repair/8820/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.683310542Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115268535Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115334667Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115360264Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115383931Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115400409Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736150926Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"6"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:07.736247164Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950,"task":"recv","id":"6"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736360551Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } (ConnectionId(6)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736385918Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } (ConnectionId(6)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736408595Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1950}