{"msg":"Opened existing region file \"/var/tmp/test_repair/8820/region.json\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.794820062Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.795372311Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.795398537Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"UUID: 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.797733984Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.797755081Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7817632486f84ab6865c5ce71e16196ec0c710fc\nCommit timestamp: 2025-01-28T21:38:04.000000000Z branch: mkeeter/early-io-rejection\nrustc: 1.84.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.841939848Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.842003139Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12820 for path:\"/var/tmp/test_repair/8820\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.842936852Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.843245046Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"local_addr":"0.0.0.0:12820","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.843375077Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.843404793Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.84342506Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"accepted connection from 127.0.0.1:62974","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.844643071Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"main"} {"msg":"connection (127.0.0.1:62974): tasks spawned","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.8448572Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"id":"0"} {"msg":"Connection request from 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a with version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.845158695Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } connected, version 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.84519464Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.845628876Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Current flush_numbers [0..12]: [352, 354, 350, 355, 356, 352, 353, 351, 350, 356, 352, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.846479191Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.847837582Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.849523504Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.851372592Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/000.copy\" to \"/var/tmp/test_repair/8820/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.851913743Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/000.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.851981113Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/000.replace\" to \"/var/tmp/test_repair/8820/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.852120322Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.853573369Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.85424626Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.855895418Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/001.copy\" to \"/var/tmp/test_repair/8820/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856520196Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/001.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856547762Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/001.replace\" to \"/var/tmp/test_repair/8820/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.856730925Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.858063169Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.858766036Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.860467546Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/01D.copy\" to \"/var/tmp/test_repair/8820/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.863054755Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/01D.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.863089001Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/01D.replace\" to \"/var/tmp/test_repair/8820/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.863212742Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Created copy dir \"/var/tmp/test_repair/8820/00/000/019.copy\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.864580741Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"eid:25 Found repair files: [\"019\"]","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.865120242Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Verify extent 25 still ready for copy","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.866662465Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8820/00/000/019.copy\" to \"/var/tmp/test_repair/8820/00/000/019.replace\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867203496Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Copy files from \"/var/tmp/test_repair/8820/00/000/019.replace\" in \"/var/tmp/test_repair/8820/00/000\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867232302Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"Move directory \"/var/tmp/test_repair/8820/00/000/019.replace\" to \"/var/tmp/test_repair/8820/00/000/019.completed\"","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:22.867356174Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"extent 0 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381762046Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"extent 1 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381819117Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381847183Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:23.381864161Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.92475266Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2025-01-28T22:04:26.924851986Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.924887221Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.924971648Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978} {"msg":"upstairs UpstairsConnection { upstairs_id: 0dd7082a-96f3-467e-91fe-cb80b1dd9f2a, session_id: ca6d00de-55cc-41bb-868e-bc5463f904ac, gen: 199 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2025-01-28T22:04:26.925007573Z","hostname":"w-01JJQEEN7EZQA0GYQQ8SEK2BJY","pid":1978}