{"msg":"Opened existing region file \"/var/tmp/test_repair/8810/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.355406657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.3558656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.355897739Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"UUID: 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.365921647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"Blocks per extent:20 Total Extents: 30","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.365955096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 311c3ca2c97d568407ae06780269d96fcfdd3a7c\nCommit timestamp: 2024-11-07T13:47:04.000000000Z branch: renovate/tokio-1.x-lockfile\nrustc: 1.82.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.408877597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.408928525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"Repair listens on 0.0.0.0:12810 for path:\"/var/tmp/test_repair/8810\"","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.40989443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"repair"} {"msg":"listening","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.410151871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"local_addr":"0.0.0.0:12810","task":"repair"} {"msg":"Using repair address: 0.0.0.0:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.410273266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.410310125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"downstairs listening on 0.0.0.0:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.410416781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"accepted connection from 127.0.0.1:57309","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465784992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"main"} {"msg":"connection (127.0.0.1:57309): tasks spawned","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465967416Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"id":"0"} {"msg":"Connection request from 4cc4c7ec-e57a-4894-8318-957169252e08 with version 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466286334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"upstairs UpstairsConnection { upstairs_id: 4cc4c7ec-e57a-4894-8318-957169252e08, session_id: 0fed1019-ebf5-422b-aff6-7aead50a9d67, gen: 199 } connected, version 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466318043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"UpstairsConnection { upstairs_id: 4cc4c7ec-e57a-4894-8318-957169252e08, session_id: 0fed1019-ebf5-422b-aff6-7aead50a9d67, gen: 199 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466814975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"Current flush_numbers [0..12]: [335, 332, 334, 334, 302, 316, 334, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467627536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.470705745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"local_addr":"0.0.0.0:12810","task":"repair","remote_addr":"127.0.0.1:60860"} {"msg":"accepted connection","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.470737634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"local_addr":"0.0.0.0:12810","task":"repair","remote_addr":"127.0.0.1:53153"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.47139637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/18/files","method":"GET","req_id":"c2b9d588-6578-4262-ae8b-a906b247f6fb","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":236,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.471441938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/18/files","method":"GET","req_id":"591304a7-b2f2-46e1-adad-46f1c5890e89","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":238,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.472364315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/18/data","method":"GET","req_id":"1b2527a9-4669-4b91-9b2b-247b05c0bd07","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":341,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.472418893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/18/data","method":"GET","req_id":"acf0a895-6088-4ce2-84b4-10605f87b2c4","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":346,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.475947926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/18/repair-ready","method":"GET","req_id":"3990b84b-e32e-4513-a677-643abb89a322","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":122,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.475988274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/18/repair-ready","method":"GET","req_id":"4875f875-2f75-4d5f-86f1-107749bdc07f","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.484720939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/3/files","method":"GET","req_id":"a279b8c2-8c23-4583-bf94-26da3f2aa940","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.484805526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/3/files","method":"GET","req_id":"60dcfc91-ad26-4ca6-b245-8e1dea13580e","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.485488441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/3/data","method":"GET","req_id":"76e4cdab-00f9-4fd4-8980-690e449cf328","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":189,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.485534669Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/3/data","method":"GET","req_id":"20260ad8-0b35-4208-83d1-2cc8e4170837","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":219,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.488747133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/3/repair-ready","method":"GET","req_id":"6b85c433-79d2-44b8-87f2-5cfa653de8aa","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.488793752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/3/repair-ready","method":"GET","req_id":"e67e554d-8cca-4023-8dc7-61101416844a","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":122,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.497025135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/13/files","method":"GET","req_id":"224f960b-29db-4703-86a7-53f4f4b38f4b","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.497226587Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/13/files","method":"GET","req_id":"f038b21e-e770-48e3-93f5-45b21cc4a619","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.497692521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/13/data","method":"GET","req_id":"577f4ee0-0579-4067-8c1e-237126db370b","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":232,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.497970891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/13/data","method":"GET","req_id":"e9ed957b-9ca5-4060-84bf-fbb39e66d461","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":225,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.501094618Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/13/repair-ready","method":"GET","req_id":"74af3818-d4ff-43a5-971c-31c134ee08f8","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.501134216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/13/repair-ready","method":"GET","req_id":"5b42183a-4d0c-4e16-9ae4-6b67a306f2d0","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.509498135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/25/files","method":"GET","req_id":"f2a304fe-ae98-4faf-92fb-62b74639f16c","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.509545723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/25/files","method":"GET","req_id":"768b3f1a-a946-4d39-9cb7-65499e714ae6","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.51016364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/25/data","method":"GET","req_id":"e8f357e5-6d96-449d-8d84-c4b877c8f522","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":219,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.510385922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/25/data","method":"GET","req_id":"263a4810-66df-47fb-9b12-f275cb3271d9","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":239,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.513668064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/25/repair-ready","method":"GET","req_id":"f0fdcb81-38ad-4691-8eee-6aab623a5106","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.513707052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/25/repair-ready","method":"GET","req_id":"b926ecb7-65a9-41c4-a13a-75c137d0e6eb","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.52208063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/2/files","method":"GET","req_id":"5fcaae2f-21f2-4976-904f-366f3c14f0b6","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":185,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.526113205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/2/files","method":"GET","req_id":"ab400dd6-0f08-4d18-aaa7-ea8e9c1bae35","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":4200,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.526206141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/2/data","method":"GET","req_id":"90314c64-6c96-4b8a-8d29-4f9895a65567","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":3498,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.526845758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/newextent/2/data","method":"GET","req_id":"4155e3da-8ab0-4a0e-9381-715c9ee96d2c","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":228,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.529609158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/2/repair-ready","method":"GET","req_id":"3356e99a-eff8-4a55-ab19-f1c7beafebb9","remote_addr":"127.0.0.1:53153","local_addr":"0.0.0.0:12810","task":"repair","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.529685896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"uri":"/extent/2/repair-ready","method":"GET","req_id":"7a308cfd-37e0-4c30-b614-2c832c52bafc","remote_addr":"127.0.0.1:60860","local_addr":"0.0.0.0:12810","task":"repair","latency_us":86,"response_code":"200"} {"msg":"extent 2 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.967299818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"extent 3 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.967359576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"extent 13 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.967388165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"extent 18 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.967411554Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"extent 25 was in dirty_extents but not actually dirty","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.967434533Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"upstairs disconnected","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:32.590471715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"recv","id":"0"} {"msg":"recv_task sending ConnectionClosed","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:32.590544173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976,"task":"recv","id":"0"} {"msg":"connection closed; disconnection","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:32.590568642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"upstairs UpstairsConnection { upstairs_id: 4cc4c7ec-e57a-4894-8318-957169252e08, session_id: 0fed1019-ebf5-422b-aff6-7aead50a9d67, gen: 199 } (ConnectionId(0)) removed, 0 jobs left","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:32.590592111Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976} {"msg":"upstairs UpstairsConnection { upstairs_id: 4cc4c7ec-e57a-4894-8318-957169252e08, session_id: 0fed1019-ebf5-422b-aff6-7aead50a9d67, gen: 199 } (ConnectionId(0)) was previously active, clearing","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:32.59061464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1976}