{"msg":"Opened existing region file \"/var/tmp/test_repair/8830/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.532089035Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.532588946Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.532614183Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"UUID: 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.541990072Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.542023338Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 664b05998b747a86e13a12bb6c5e4428458d8bd9\nCommit timestamp: 2024-10-31T20:43:19.000000000Z branch: alan/crutest-volume-info\nrustc: 1.80.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.586210031Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.586261125Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12830 for path:\"/var/tmp/test_repair/8830\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.587237049Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.587554711Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"local_addr":"0.0.0.0:12830","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.587719842Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.587753898Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.587769506Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"accepted connection from 127.0.0.1:34399","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614359184Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"main"} {"msg":"connection (127.0.0.1:34399): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614694075Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"id":"0"} {"msg":"Connection request from a4789ed1-39f5-432e-8d86-819e12466435 with version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614921078Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614951604Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615482531Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Current flush_numbers [0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 346]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616500191Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.619689853Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.621348406Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.624247132Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/003.copy\" to \"/var/tmp/test_repair/8830/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.624832673Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/003.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.626558048Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/003.replace\" to \"/var/tmp/test_repair/8830/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.628265176Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/012.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.63363644Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"eid:18 Found repair files: [\"012\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.634192784Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Verify extent 18 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.638827484Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/012.copy\" to \"/var/tmp/test_repair/8830/00/000/012.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.639543549Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/012.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.640622141Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/012.replace\" to \"/var/tmp/test_repair/8830/00/000/012.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.644344351Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/01D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.649616076Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"eid:29 Found repair files: [\"01D\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.650155692Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Verify extent 29 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.654026663Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/01D.copy\" to \"/var/tmp/test_repair/8830/00/000/01D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.65472615Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/01D.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.656313612Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/01D.replace\" to \"/var/tmp/test_repair/8830/00/000/01D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.657776168Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/011.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.662883653Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"eid:17 Found repair files: [\"011\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.66341288Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Verify extent 17 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.667424835Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/011.copy\" to \"/var/tmp/test_repair/8830/00/000/011.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.667982679Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/011.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.668972681Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/011.replace\" to \"/var/tmp/test_repair/8830/00/000/011.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.67050161Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Created copy dir \"/var/tmp/test_repair/8830/00/000/010.copy\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.675863855Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"eid:16 Found repair files: [\"010\"]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.67640689Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Verify extent 16 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.679747044Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"1 repair files downloaded, move directory \"/var/tmp/test_repair/8830/00/000/010.copy\" to \"/var/tmp/test_repair/8830/00/000/010.replace\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.680281551Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Copy files from \"/var/tmp/test_repair/8830/00/000/010.replace\" in \"/var/tmp/test_repair/8830/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.681087445Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"Move directory \"/var/tmp/test_repair/8830/00/000/010.replace\" to \"/var/tmp/test_repair/8830/00/000/010.completed\"","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.683352007Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115291622Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"extent 16 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115340036Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"extent 17 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115365423Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.11538982Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"extent 29 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:04.115413618Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736247204Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:07.73636446Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736390637Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736413794Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974} {"msg":"upstairs UpstairsConnection { upstairs_id: a4789ed1-39f5-432e-8d86-819e12466435, session_id: 033ae411-3ae5-402e-b062-bad869f41cb8, gen: 199 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:07.736443951Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1974}