/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.535765589Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.58106677Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.581674288Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"664b05998b747a86e13a12bb6c5e4428458d8bd9\",\n git_commit_timestamp: \"2024-10-31T20:43:19.000000000Z\",\n git_branch: \"alan/crutest-volume-info\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.581710124Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.581724422Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.581739011Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} {"msg":"Crucible a4789ed1-39f5-432e-8d86-819e12466435 has session id: 033ae411-3ae5-402e-b062-bad869f41cb8","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.582209435Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Upstairs opts: Upstairs UUID: a4789ed1-39f5-432e-8d86-819e12466435, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.582255749Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Crucible stats registered with UUID: a4789ed1-39f5-432e-8d86-819e12466435","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.582279087Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61396964Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614005756Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} {"msg":"a4789ed1-39f5-432e-8d86-819e12466435 active request set","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61406141Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614082307Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614111844Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614134811Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614155029Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614172316Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614189354Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614206012Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614223461Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614240668Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34399)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614257986Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:44213)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614461492Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57015)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.614589547Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615117515Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615146321Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615164259Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615189076Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615216903Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615291254Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615848158Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.615985961Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61608384Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616388884Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616410131Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616424409Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616577691Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616598069Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616611727Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61667018Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616689118Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[0]R flush_numbers[0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 346]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616743732Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[0]R generation[0..12]: [192, 197, 197, 182, 191, 194, 194, 196, 197, 192, 192, 198]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616764259Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616779618Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[1]R flush_numbers[0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 346]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616797565Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[1]R generation[0..12]: [192, 197, 197, 182, 191, 194, 194, 196, 197, 192, 192, 198]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616812354Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616826632Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[2]R flush_numbers[0..12]: [336, 344, 344, 321, 335, 341, 341, 343, 344, 337, 338, 346]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61684033Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[2]R generation[0..12]: [192, 197, 197, 182, 191, 194, 194, 196, 197, 192, 192, 198]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616854179Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, true, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616868267Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616881366Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616894684Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616907732Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616920501Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616934329Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:3 gens: 182 182 182","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616948738Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:3 flush: 321 321 321 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616963336Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616977924Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:3 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.616994102Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61700877Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617023089Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617037377Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Extents 16 dirty","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617058225Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"First source client ID for extent 16","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617072903Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:16 gens: 187 187 187","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617087541Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:16 flush: 329 329 329 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617102489Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617117118Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:16 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617141095Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"find dest for source 0 for extent at index 16","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617168661Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617184419Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617199038Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Extents 17 dirty","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617213336Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"First source client ID for extent 17","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617227145Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:17 gens: 187 187 187","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617244003Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:17 flush: 329 329 329 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617259011Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617273859Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:17 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617288217Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"find dest for source 0 for extent at index 17","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617302736Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617317054Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617331322Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Extents 18 dirty","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61734549Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"First source client ID for extent 18","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617359259Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:18 gens: 182 182 182","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617373287Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:18 flush: 321 321 321 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617388095Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617402774Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:18 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617417192Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"find dest for source 0 for extent at index 18","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61743159Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617447478Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617462106Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Extents 29 dirty","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617476155Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"First source client ID for extent 29","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617489863Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:29 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617504082Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:29 flush: 343 343 343 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61751886Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617533328Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"extent:29 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617547667Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"find dest for source 0 for extent at index 29","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617561855Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617575893Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617590341Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.61760486Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617618328Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617631457Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"starting reconciliation 9417d68b-1297-4061-b2d1-1f9ff140553b: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617645315Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Full repair list: {ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(18): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(29): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(17): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(16): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617665202Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 348, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(18), client_id: ClientId(0), flush_number: 348, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(18) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(18), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(18) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(29), client_id: ClientId(0), flush_number: 348, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(29) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(29), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(29) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(17), client_id: ClientId(0), flush_number: 348, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(17) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(17), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(17) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(16), client_id: ClientId(0), flush_number: 348, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(16) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(16), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(16) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.617741513Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.617757682Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.61777306Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.619138088Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.619398007Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.619436663Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.61954576Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.631405184Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.631833823Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.633096764Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.633397198Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.633429604Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.633491667Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.647310779Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.64772683Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.649106606Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.649383733Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.649434367Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.649473903Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.660482698Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.660929715Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.662334088Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.662628263Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.662665869Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.662699185Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.673347553Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.673785031Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.675335757Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.675636422Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.675675097Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.675708583Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.685989824Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.68636344Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686393367Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"0","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686409775Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"1","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686424123Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"client":"2","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686438211Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.68645114Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"a4789ed1-39f5-432e-8d86-819e12466435 is now active with session: 033ae411-3ae5-402e-b062-bad869f41cb8","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686464468Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-10-31T21:08:03.686481616Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976} Wait for a query_work_queue command to finish before sending IO {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-10-31T21:08:03.686566566Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976,"reconcile":"9417d68b-1297-4061-b2d1-1f9ff140553b","":"downstairs","":"downstairs","session_id":"033ae411-3ae5-402e-b062-bad869f41cb8"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-10-31T21:08:07.735809326Z","hostname":"w-01JBJ60HQJPRWN8XY4C1QDEXQW","pid":1976}