/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.432773326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.478025565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.478961556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479005699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479021817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479048694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} {"msg":"Crucible e8142854-0167-4302-b987-443c59c07ad1 has session id: e4c1e002-e6ef-4e58-a5d1-89352705f1ac","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479573681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Upstairs opts: Upstairs UUID: e8142854-0167-4302-b987-443c59c07ad1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479607016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Crucible stats registered with UUID: e8142854-0167-4302-b987-443c59c07ad1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.479623884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512224841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} {"msg":"e8142854-0167-4302-b987-443c59c07ad1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512268465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512284663Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512312079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512345734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51237319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512390028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512411335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512437301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512462118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512489174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:51960)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512515041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52351)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512540507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:60438)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.512792003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"io task","client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.513356005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.513383181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51353593Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51361043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.513648604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51367905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514020973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514142896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51418284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514611731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514650986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514679772Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514747202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514775079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514789797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514802665Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514816193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[0]R flush_numbers[0..12]: [334, 325, 334, 325, 335, 334, 324, 335, 335, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514830081Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[0]R generation[0..12]: [196, 189, 196, 189, 198, 196, 188, 198, 198, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514843839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[0]R dirty[0..12]: [false, false, true, true, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514857827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[1]R flush_numbers[0..12]: [334, 325, 334, 325, 335, 334, 324, 335, 335, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514871605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[1]R generation[0..12]: [196, 189, 196, 189, 198, 196, 188, 198, 198, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514889243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[1]R dirty[0..12]: [false, false, true, true, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514903691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[2]R flush_numbers[0..12]: [334, 325, 334, 325, 335, 334, 324, 335, 335, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514917339Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[2]R generation[0..12]: [196, 189, 196, 189, 198, 196, 188, 198, 198, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514930877Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"[2]R dirty[0..12]: [false, false, true, true, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514944615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514957593Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514970652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51498349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.514996638Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515010316Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:2 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515024554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:2 flush: 334 334 334 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515050891Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515066368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:2 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515080916Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515098474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515113122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51512711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515154296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515168894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:3 gens: 189 189 189","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515183042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:3 flush: 325 325 325 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51519768Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515211988Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:3 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515226176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515240324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515289018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515325862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Extents 6 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51534384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515358498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:6 gens: 188 188 188","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515375746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:6 flush: 324 324 324 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515390853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515405391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:6 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.5154197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"find dest for source 0 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515434347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515448616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515462754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Extents 13 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515477102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"First source client ID for extent 13","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.5154909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:13 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515505238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:13 flush: 335 335 335 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515519766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515534214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:13 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515548252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"find dest for source 0 for extent at index 13","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51556248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515576788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515592726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Extents 16 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515607134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"First source client ID for extent 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515620592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:16 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51563454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:16 flush: 334 334 334 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515649088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515663486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"extent:16 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515677544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"find dest for source 0 for extent at index 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515691882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.51570587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515719918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515734416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515747884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515760913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"starting reconciliation b1eb42d4-8e55-48ae-8171-30d202fa9308: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515774771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Full repair list: {ExtentId(16): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(13): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(6): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515794988Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(16), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(16) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(16), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(16) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(13), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(13) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(13), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(13) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(6), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(6), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 336, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.515867958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.515883165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.515898283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.517015709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.517338515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.517377519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.517407855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.526879347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.52728921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.528621006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.528897428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.528937572Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.528973097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.53757298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.537948118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.538992004Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.539294132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.539319898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.539342995Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.548918972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.54929569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.550177068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.55045792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.550483956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.550507233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.559834854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.560208833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.561999565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.562224314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.56225228Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.562270138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.570843174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571142782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571170368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"0","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571189676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"1","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571206683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"client":"2","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571223411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571238699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"e8142854-0167-4302-b987-443c59c07ad1 is now active with session: e4c1e002-e6ef-4e58-a5d1-89352705f1ac","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571254767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.571269925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977} Wait for a query_work_queue command to finish before sending IO {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T03:57:43.571353323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977,"reconcile":"b1eb42d4-8e55-48ae-8171-30d202fa9308","":"downstairs","":"downstairs","session_id":"e4c1e002-e6ef-4e58-a5d1-89352705f1ac"} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Read 88 Done Done Done false 2 Acked 1001 Read 256 Done Done Done false 3 Acked 1002 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 3 2 1 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 4 3 2 1 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:47.624394631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1977}