/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.368647333Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.413516418Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.414678718Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d77aa90b6afe7d7ddb1dc63211fa4216c8866a90\",\n git_commit_timestamp: \"2025-01-27T19:20:16.000000000Z\",\n git_branch: \"mkeeter/shrink-replay-buffer\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.41474116Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.414764885Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.414787441Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} {"msg":"Crucible d7f46382-ea9f-4c1b-9e50-89236223a8d2 has session id: be6d5c53-937c-4b85-af20-6e75594d00fc","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.415468459Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Upstairs opts: Upstairs UUID: d7f46382-ea9f-4c1b-9e50-89236223a8d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.415502648Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Crucible stats registered with UUID: d7f46382-ea9f-4c1b-9e50-89236223a8d2","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.415517919Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:44:55.4155306Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.415963013Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.415985998Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} {"msg":"d7f46382-ea9f-4c1b-9e50-89236223a8d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416054956Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416071905Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416099328Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416121865Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416148558Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416175192Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416204853Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416228918Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416253233Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416278887Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:64545)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416303392Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35880)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416492045Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43269)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.416524355Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.4171624Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.417199537Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.417228959Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.417756742Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.417859538Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.417887591Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418347776Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418448564Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[0]R flush_numbers[0..12]: [347, 349, 351, 359, 357, 357, 347, 359, 355, 358, 358, 356]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418522947Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[0]R generation[0..12]: [191, 193, 194, 198, 197, 197, 191, 198, 196, 198, 198, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418552429Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[0]R dirty[0..12]: [false, true, true, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.41856803Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[1]R flush_numbers[0..12]: [347, 349, 351, 359, 357, 357, 347, 359, 355, 358, 358, 356]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418581721Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[1]R generation[0..12]: [191, 193, 194, 198, 197, 197, 191, 198, 196, 198, 198, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418595243Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[1]R dirty[0..12]: [false, true, true, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418609434Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[2]R flush_numbers[0..12]: [347, 349, 351, 359, 357, 357, 347, 359, 355, 358, 358, 356]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418623056Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[2]R generation[0..12]: [191, 193, 194, 198, 197, 197, 191, 198, 196, 198, 198, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418636367Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"[2]R dirty[0..12]: [false, true, true, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418649729Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418667348Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.4186811Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Next flush: 360","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418693582Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent 1 needs reconciliation: ExtentMetadata { gen: 193, flush: 349, dirty: true } ExtentMetadata { gen: 193, flush: 349, dirty: true } ExtentMetadata { gen: 193, flush: 349, dirty: true }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418708343Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent:0 [ExtentMetadata { gen: 193, flush: 349, dirty: true }, ExtentMetadata { gen: 193, flush: 349, dirty: true }, ExtentMetadata { gen: 193, flush: 349, dirty: true }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418724073Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"found dest for source 0 for extent at index 1 => [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418737845Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent 2 needs reconciliation: ExtentMetadata { gen: 194, flush: 351, dirty: true } ExtentMetadata { gen: 194, flush: 351, dirty: true } ExtentMetadata { gen: 194, flush: 351, dirty: true }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418752955Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent:0 [ExtentMetadata { gen: 194, flush: 351, dirty: true }, ExtentMetadata { gen: 194, flush: 351, dirty: true }, ExtentMetadata { gen: 194, flush: 351, dirty: true }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418810829Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"found dest for source 0 for extent at index 2 => [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418826959Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent 15 needs reconciliation: ExtentMetadata { gen: 194, flush: 351, dirty: true } ExtentMetadata { gen: 194, flush: 351, dirty: true } ExtentMetadata { gen: 194, flush: 351, dirty: true }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.41884238Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent:0 [ExtentMetadata { gen: 194, flush: 351, dirty: true }, ExtentMetadata { gen: 194, flush: 351, dirty: true }, ExtentMetadata { gen: 194, flush: 351, dirty: true }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.41885818Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"found dest for source 0 for extent at index 15 => [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418872441Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent 24 needs reconciliation: ExtentMetadata { gen: 192, flush: 348, dirty: true } ExtentMetadata { gen: 192, flush: 348, dirty: true } ExtentMetadata { gen: 192, flush: 348, dirty: true }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418892039Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent:0 [ExtentMetadata { gen: 192, flush: 348, dirty: true }, ExtentMetadata { gen: 192, flush: 348, dirty: true }, ExtentMetadata { gen: 192, flush: 348, dirty: true }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418907369Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"found dest for source 0 for extent at index 24 => [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418921071Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent 26 needs reconciliation: ExtentMetadata { gen: 197, flush: 357, dirty: true } ExtentMetadata { gen: 197, flush: 357, dirty: true } ExtentMetadata { gen: 197, flush: 357, dirty: true }","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418936002Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"extent:0 [ExtentMetadata { gen: 197, flush: 357, dirty: true }, ExtentMetadata { gen: 197, flush: 357, dirty: true }, ExtentMetadata { gen: 197, flush: 357, dirty: true }] => 0","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418950913Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"found dest for source 0 for extent at index 26 => [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418964494Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.418984702Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419014673Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419036509Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Full repair list: {ExtentId(15): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(26): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(24): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419071068Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(15), client_id: ClientId(0), flush_number: 360, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(15) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(15), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(15) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 360, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 360, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(26), client_id: ClientId(0), flush_number: 360, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(26) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(26), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(26) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(24), client_id: ClientId(0), flush_number: 360, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(24) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(24), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(24) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419202587Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"starting reconciliation 761137d9-9501-4a3c-ae6d-169068036353: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419231709Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.419339082Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.41935783Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.425650404Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.425696296Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.430188164Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.430228899Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.4397062Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.439740728Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.444025255Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.444062792Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.44755512Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"":"downstairs","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447583422Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"0","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447597993Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"1","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447611495Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"client":"2","":"downstairs","session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447624537Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447641276Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"d7f46382-ea9f-4c1b-9e50-89236223a8d2 is now active with session: be6d5c53-937c-4b85-af20-6e75594d00fc","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.447654618Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978,"session_id":"be6d5c53-937c-4b85-af20-6e75594d00fc"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2025-01-27T19:44:55.44766677Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-27T19:44:59.500918541Z","hostname":"w-01JJMM4YQC45QTGBDP0DKQX50W","pid":1978}