/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.264529569Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.307665095Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308252763Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9e483d552714982cf7bdcde4e93a76c1673dc94b\",\n git_commit_timestamp: \"2024-11-01T04:11:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308296418Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308311537Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308325795Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"Crucible 91a4d621-d4c9-4392-b39b-560e490b847b has session id: 0073dd67-57aa-4fd5-abab-bc8aeefb9794","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308762289Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Upstairs opts: Upstairs UUID: 91a4d621-d4c9-4392-b39b-560e490b847b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308806274Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Crucible stats registered with UUID: 91a4d621-d4c9-4392-b39b-560e490b847b","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.308824422Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338589246Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338618723Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"91a4d621-d4c9-4392-b39b-560e490b847b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338693855Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338716322Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338754178Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338794694Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.33883051Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338871186Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338905373Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338939259Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338961296Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.338992023Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_connection connected from Ok(127.0.0.1:34047)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339026469Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_connection connected from Ok(127.0.0.1:39852)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339056906Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_connection connected from Ok(127.0.0.1:33472)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339179713Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"io task","client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339843573Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.33986876Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339895608Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339909316Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339934363Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.339951311Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.340494154Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.340604992Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.340680564Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341044176Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341064904Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341085661Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341146795Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341175532Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.34119108Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341204259Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341218207Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[0]R flush_numbers[0..12]: [310, 351, 350, 344, 343, 348, 341, 323, 335, 350, 332, 340]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341239065Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[0]R generation[0..12]: [176, 198, 198, 195, 194, 196, 194, 183, 190, 198, 188, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341253553Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341267602Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[1]R flush_numbers[0..12]: [310, 351, 350, 344, 343, 348, 341, 323, 335, 350, 332, 340]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.3412852Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[1]R generation[0..12]: [176, 198, 198, 195, 194, 196, 194, 183, 190, 198, 188, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341299548Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341313667Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[2]R flush_numbers[0..12]: [310, 351, 350, 344, 343, 348, 341, 323, 335, 350, 332, 340]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341327496Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[2]R generation[0..12]: [176, 198, 198, 195, 194, 196, 194, 183, 190, 198, 188, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341341384Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, true, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341355203Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341368411Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.34138187Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341394929Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Extents 5 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341407687Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341421276Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:5 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341435574Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:5 flush: 348 348 348 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341450432Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341465151Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:5 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341481069Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341495548Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341509716Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341523895Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Extents 27 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.34156666Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"First source client ID for extent 27","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341583698Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:27 gens: 194 194 194","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341598167Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:27 flush: 341 341 341 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341612995Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341627484Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"extent:27 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341641452Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"find dest for source 0 for extent at index 27","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341655631Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341669869Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341683868Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341705305Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341719344Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341734732Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"starting reconciliation 2d9550c0-2535-415d-9810-e38fcc47c2d7: found 2 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341748831Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Full repair list: {ExtentId(5): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(27): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341764239Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(5), client_id: ClientId(0), flush_number: 352, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(5) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(5), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(5) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(27), client_id: ClientId(0), flush_number: 352, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(27) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(27), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(27) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.341796586Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.341810704Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.341827663Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.342927416Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.343195258Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.343238923Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.343270799Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.355100115Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.355472945Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.356929521Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.357239108Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.357274805Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.357295432Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.367811197Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368124524Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.3681607Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"0","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368180838Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"1","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368198236Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"client":"2","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368215234Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368234402ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"91a4d621-d4c9-4392-b39b-560e490b847b is now active with session: 0073dd67-57aa-4fd5-abab-bc8aeefb9794","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368263589Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-11-01T04:40:12.368293056Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981} {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T04:40:12.368319743Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981,"reconcile":"2d9550c0-2535-415d-9810-e38fcc47c2d7","":"downstairs","":"downstairs","session_id":"0073dd67-57aa-4fd5-abab-bc8aeefb9794"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T04:40:16.421351232Z","hostname":"w-01JBJZW2F0X5YAPNVHPCRTK4NC","pid":1981}