/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.088611997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.133964717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.134622805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.134652563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.134665803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.134679772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"Crucible 72ee9730-01a5-4613-b02f-1389b7821f55 has session id: 7dc51e0d-725b-4e05-ade2-363000c16232","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.135239324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Upstairs opts: Upstairs UUID: 72ee9730-01a5-4613-b02f-1389b7821f55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.135285662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Crucible stats registered with UUID: 72ee9730-01a5-4613-b02f-1389b7821f55","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.135309601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166071818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"task":"crutest"} {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166104747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"72ee9730-01a5-4613-b02f-1389b7821f55 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166120276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166136115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166163744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166185723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166203172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166219731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16623623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166252819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166270009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166286798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_connection connected from Ok(127.0.0.1:54384)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166383753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_connection connected from Ok(127.0.0.1:47043)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.166420991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_connection connected from Ok(127.0.0.1:55769)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16644697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"io task","client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167180154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167210003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167226042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167240291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16725375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167272179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.167959366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168005203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168039562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168530218Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168559306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168574495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168587355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168601284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168614623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168627043Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168640642Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 356, 356, 345, 355, 355, 356, 356, 356, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168654701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[0]R generation[0..12]: [197, 197, 198, 198, 192, 197, 197, 198, 198, 198, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168668711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16868276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 356, 356, 345, 355, 355, 356, 356, 356, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168700299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[1]R generation[0..12]: [197, 197, 198, 198, 192, 197, 197, 198, 198, 198, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168714618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168728678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 356, 356, 345, 355, 355, 356, 356, 356, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168742407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[2]R generation[0..12]: [197, 197, 198, 198, 192, 197, 197, 198, 198, 198, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168756027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168769826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168782725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168795905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168808834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Extents 12 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168821793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"First source client ID for extent 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168835392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:12 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168849732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:12 flush: 356 356 356 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168864291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16887866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:12 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16889453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"find dest for source 0 for extent at index 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168909139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168923608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168937698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Extents 26 dirty","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168962236Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"First source client ID for extent 26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.168977116Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:26 gens: 198 198 198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169011064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:26 flush: 356 356 356 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169029313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169044172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"extent:26 dirty: true true true","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169058301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"find dest for source 0 for extent at index 26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169072631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.16908678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169100899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169115819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169129318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169144907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Full repair list: {ExtentId(12): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(26): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169160927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(12), client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(12) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(12), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(12) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(26), client_id: ClientId(0), flush_number: 358, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(26) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(26), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(26) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169193665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"starting reconciliation 61c1a305-fa69-4360-883b-cfae80d8ca2a: found 8 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169208554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.169222173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.169240612Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.169262902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169617834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.169648283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.169663222Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.174614908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.175036498Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.175317604Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.175629109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.175669847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.175702055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.179520427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179842891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.17987161Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"0","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179887349Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"1","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179901468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"client":"2","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179915298Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179931237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"72ee9730-01a5-4613-b02f-1389b7821f55 is now active with session: 7dc51e0d-725b-4e05-ade2-363000c16232","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179944656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"session_id":"Wait for a query_work_queue command to finish before sending IO 7dc51e0d-725b-4e05-ade2-363000c16232"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.179965835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978} {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-26T22:11:39.179977255Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978,"reconcile":"61c1a305-fa69-4360-883b-cfae80d8ca2a","":"downstairs","":"downstairs","session_id":"7dc51e0d-725b-4e05-ade2-363000c16232"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Read 88 Done Done Done false 1001 Acked Read 256 Done Done Done false 1002 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1002 1001 1000 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1003 1002 1001 1000 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:43.230268111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1978}