/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.359394403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.40130103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.401695036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.434407585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.434933776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.434969605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.434983894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.435002733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} {"msg":"Crucible 4cc4c7ec-e57a-4894-8318-957169252e08 has session id: 0fed1019-ebf5-422b-aff6-7aead50a9d67","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.435434458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Upstairs opts: Upstairs UUID: 4cc4c7ec-e57a-4894-8318-957169252e08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.435477816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Crucible stats registered with UUID: 4cc4c7ec-e57a-4894-8318-957169252e08","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.435496916Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465280051Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465325849Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} {"msg":"4cc4c7ec-e57a-4894-8318-957169252e08 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465363888Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465401216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465449565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465488643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465524582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.46555953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465594499Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465620598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465655027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465687586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:37768)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465721865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:57309)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465788292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:41519)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.465825181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466497537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466526666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466547455Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466564794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466579644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.466592983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467143953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467280308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467422203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467843828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467872507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467887877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467901046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467914825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467928385Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467940724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467954184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[0]R flush_numbers[0..12]: [335, 332, 334, 334, 302, 316, 334, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467968203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[0]R generation[0..12]: [198, 197, 198, 198, 178, 187, 198, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467982253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[0]R dirty[0..12]: [false, false, true, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.467999482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[1]R flush_numbers[0..12]: [335, 332, 334, 334, 302, 316, 334, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468013882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[1]R generation[0..12]: [198, 197, 198, 198, 178, 187, 198, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468027832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[1]R dirty[0..12]: [false, false, true, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468041611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[2]R flush_numbers[0..12]: [335, 332, 334, 334, 302, 316, 334, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468055271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[2]R generation[0..12]: [198, 197, 198, 198, 178, 187, 198, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"[2]R dirty[0..12]: [false, false, true, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468083039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468096229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468109549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468122758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468135957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468150807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:2 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468165496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:2 flush: 334 334 334 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468180396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468196875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:2 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468211295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468225574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468239724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468253833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468276803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468291492Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:3 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468306471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:3 flush: 334 334 334 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468321141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.4683355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:3 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.46834979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468364379Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468378489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468392588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Extents 13 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468408788Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"First source client ID for extent 13","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468422917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:13 gens: 187 187 187","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468437197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:13 flush: 316 316 316 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468451756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468466226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:13 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468480415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"find dest for source 0 for extent at index 13","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468494615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468508734Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468522774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Extents 18 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468536793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"First source client ID for extent 18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468550393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:18 gens: 190 190 190","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468564312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:18 flush: 322 322 322 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468579001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468593541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:18 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468609111Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"find dest for source 0 for extent at index 18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.46862378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468637919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468652019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Extents 25 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468666108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"First source client ID for extent 25","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468679728Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:25 gens: 195 195 195","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468693797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:25 flush: 329 329 329 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468708237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468722546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"extent:25 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468736666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"find dest for source 0 for extent at index 25","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468750915Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468765105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468779204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468793754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468807293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468835232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"starting reconciliation fe723c4e-c850-4b02-999e-aba32521a561: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468853202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Full repair list: {ExtentId(18): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(13): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(25): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468871521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(18), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(18) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(18), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(18) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(13), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(13) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(13), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(13) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(25), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(25) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(25), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(25) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.468935429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.468950288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.468965478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.469843106Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.470139825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.470178584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.470210193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.482040995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.482607235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.483869369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.48414436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.484185478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.484246206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.49437103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.494788935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.496202944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.496481684Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.496517333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.496548192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.506940247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.507342782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.508636446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.508870347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.508911006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.508948734Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.519415497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.519821952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.521153264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.521407974Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.521444123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.521503321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.535322262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.53565823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535688339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"0","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535704638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"1","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535718898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"client":"2","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535732728Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535749297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"4cc4c7ec-e57a-4894-8318-957169252e08 is now active with session: 0fed1019-ebf5-422b-aff6-7aead50a9d67","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535762686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.535775176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978} Wait for a query_work_queue command to finish before sending IO {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-07T14:10:28.535857213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1978,"reconcile":"fe723c4e-c850-4b02-999e-aba32521a561","":"downstairs","":"downstairs","session_id":"0fed1019-ebf5-422b-aff6-7aead50a9d67"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program