{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.656053132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.656495775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.656527989Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.659804959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.659833184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\nCommit timestamp: 2024-09-27T16:52:31.000000000Z branch: update-omicron-common\nrustc: 1.80.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.703804502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.703857323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.704945444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.705269018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.705464764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.70548846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.70554328Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"accepted connection from 127.0.0.1:40806","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.737752775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"main"} {"msg":"connection (127.0.0.1:40806): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.738111592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"id":"0"} {"msg":"Connection request from 812cf555-da7c-4be6-93b0-db3e769147d2 with version 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.738344282Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"upstairs UpstairsConnection { upstairs_id: 812cf555-da7c-4be6-93b0-db3e769147d2, session_id: 486d52d9-3a56-4139-a00c-47b60ac80c9d, gen: 199 } connected, version 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.738431347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"UpstairsConnection { upstairs_id: 812cf555-da7c-4be6-93b0-db3e769147d2, session_id: 486d52d9-3a56-4139-a00c-47b60ac80c9d, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.738951666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Current flush_numbers [0..12]: [362, 363, 363, 351, 362, 362, 367, 367, 362, 341, 367, 367]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.739930966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.742737427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.744521277Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.747860196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.748519621Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.749743558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.751907312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01C.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.756717485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:28 Found repair files: [\"01C\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.757450747Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 28 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.760857214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01C.copy\" to \"/var/tmp/test_repair/8820/00/000/01C.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.761594206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01C.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.762934713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01C.replace\" to \"/var/tmp/test_repair/8820/00/000/01C.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.764549292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.769547972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.770212106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.773794563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.774323471Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.775718668Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.777520434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.782157427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.782785258Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.786077625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/012.copy\" to \"/var/tmp/test_repair/8820/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.786812907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/012.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.787974865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/012.replace\" to \"/var/tmp/test_repair/8820/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.789677469Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/016.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.794979536Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:22 Found repair files: [\"016\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.799907378Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 22 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.803625971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/016.copy\" to \"/var/tmp/test_repair/8820/00/000/016.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.80420397Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/016.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.806674671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/016.replace\" to \"/var/tmp/test_repair/8820/00/000/016.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.810111902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.818200955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:27 Found repair files: [\"01B\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.818912331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 27 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.824111836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01B.copy\" to \"/var/tmp/test_repair/8820/00/000/01B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.824763733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01B.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.826655834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01B.replace\" to \"/var/tmp/test_repair/8820/00/000/01B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.827964016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.838544335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.839276237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.84506232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/00E.copy\" to \"/var/tmp/test_repair/8820/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.845772197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/00E.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.847374028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/00E.replace\" to \"/var/tmp/test_repair/8820/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.848564471Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.238938528Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.238994079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 14 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.239025303Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.239051669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 22 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.239071775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 27 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.239142873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"extent 28 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:30.239165119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:33.906547971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:33.90666895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:33.906698594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"upstairs UpstairsConnection { upstairs_id: 812cf555-da7c-4be6-93b0-db3e769147d2, session_id: 486d52d9-3a56-4139-a00c-47b60ac80c9d, gen: 199 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:33.90672562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979} {"msg":"upstairs UpstairsConnection { upstairs_id: 812cf555-da7c-4be6-93b0-db3e769147d2, session_id: 486d52d9-3a56-4139-a00c-47b60ac80c9d, gen: 199 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:33.906751655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1979}