/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.202103842Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.246366171Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.24698487Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"42687c1346c9f5be1faf41362247db1201c426a8\",\n git_commit_timestamp: \"2025-01-14T15:54:39.000000000Z\",\n git_branch: \"mkeeter/separate-notify-task\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.247036813Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.24705479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.247072928Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} {"msg":"Crucible 371a3307-d72f-43ad-832c-91a61fcb537f has session id: 9253d368-6d4b-46a4-a2ba-320cc2011811","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.24767711Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Upstairs opts: Upstairs UUID: 371a3307-d72f-43ad-832c-91a61fcb537f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.247720253Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Crucible stats registered with UUID: 371a3307-d72f-43ad-832c-91a61fcb537f","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.247736811Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:45.247749399Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248163929Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248190265Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} {"msg":"371a3307-d72f-43ad-832c-91a61fcb537f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248249856Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248269513Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.24829555Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248310697Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248324765Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248337764Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248351402Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.24836419Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248377428Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248390436Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"ds_connection connected from Ok(127.0.0.1:41734)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248529965Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"ds_connection connected from Ok(127.0.0.1:38255)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248572749Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"ds_connection connected from Ok(127.0.0.1:35962)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.248602215Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.249309882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.249339728Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.249362604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.249936051Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250018369Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250041105Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250570019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250681302Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[0]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 326, 326, 325, 318, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250758891Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[0]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 198, 198, 197, 194, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250778218Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250792466Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[1]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 326, 326, 325, 318, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250806034Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[1]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 198, 198, 197, 194, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250819282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25083291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[2]R flush_numbers[0..12]: [318, 317, 325, 316, 318, 325, 318, 326, 326, 325, 318, 326]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250846638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[2]R generation[0..12]: [194, 193, 197, 192, 194, 197, 194, 198, 198, 197, 194, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250860306Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, true, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250873924Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250889812Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25090291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250915638Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Extents 6 dirty","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250928047Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250941225Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:6 gens: 194 194 194","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250954832Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:6 flush: 318 318 318 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250969121Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.250983479Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:6 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251001006Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"find dest for source 0 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251024882Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25103999Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251053848Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Extents 20 dirty","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251073405Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"First source client ID for extent 20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251087323Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:20 gens: 197 197 197","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251101231Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:20 flush: 325 325 325 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251119199Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251133826Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:20 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251147604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"find dest for source 0 for extent at index 20","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251161413Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251174931Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251188689Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Extents 22 dirty","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251202307Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"First source client ID for extent 22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251215425Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:22 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251229063Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:22 flush: 324 324 324 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251243291Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251257319Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:22 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251270827Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"find dest for source 0 for extent at index 22","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251284644Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251298422Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25131442Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Extents 23 dirty","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251328278Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"First source client ID for extent 23","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251341416Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:23 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251354804Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:23 flush: 326 326 326 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251369132Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25138314Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"extent:23 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251397068Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"find dest for source 0 for extent at index 23","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251411056Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251424604Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251438282Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251463019Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251478027Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251491365Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Full repair list: {ExtentId(22): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(6): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(20): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(23): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251508352Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(22), client_id: ClientId(0), flush_number: 327, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(22) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(22), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(22) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(6), client_id: ClientId(0), flush_number: 327, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(6), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(6) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(20), client_id: ClientId(0), flush_number: 327, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(20) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(20), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(20) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(23), client_id: ClientId(0), flush_number: 327, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(23) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(23), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(23) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251569543Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"starting reconciliation 61e15e57-8077-405a-b062-5e20da534147: found 16 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251584881Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251753666Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.251775643Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.257529956Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.25756981Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.262272076Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.262315379Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.266721908Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.266757693Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270708138Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"":"downstairs","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270735914Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"0","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270750862Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"1","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.27076408Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"client":"2","":"downstairs","session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270777268Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270789366Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"371a3307-d72f-43ad-832c-91a61fcb537f is now active with session: 9253d368-6d4b-46a4-a2ba-320cc2011811","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270801964Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978,"session_id":"9253d368-6d4b-46a4-a2ba-320cc2011811"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2025-01-14T16:18:45.270822181Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-14T16:18:49.324337932Z","hostname":"w-01JHJS736YR4R5MK8RCH140N8B","pid":1978}