/var/tmp/bins/crutest verify -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /tmp/test_repair_verify.data --verify-in /tmp/test_repair_verify.data --range -q -g 199 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.965588294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966120106Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957} {"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966148312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957} {"msg":"Crucible b5c145c1-8fd6-4009-8274-4d32fbf32d32 has session id: 8682c1e8-4554-4ebe-827b-2f0b5f01de12","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966168896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Upstairs opts: Upstairs UUID: b5c145c1-8fd6-4009-8274-4d32fbf32d32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966203081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Crucible stats registered with UUID: b5c145c1-8fd6-4009-8274-4d32fbf32d32","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966225978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:199","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966465984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957} {"msg":"b5c145c1-8fd6-4009-8274-4d32fbf32d32 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966517681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966563655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966596078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966618644Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966655496Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.96669157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966739817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.966784384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.96682689Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.967032341Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.967113968Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.967235252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.967298187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"io task","client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.968423008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.968464078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.968498583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.968521859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.96855199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.968574917Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.969474538Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.969671429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.969710387Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970311348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970371339Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970417361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970538787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970569827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970592272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970742765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97077273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[0]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970802972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[0]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.9708464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[0]R dirty[0..12]: [true, false, false, false, true, false, false, false, true, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970872079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[1]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970895437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[1]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970918599Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970947858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[2]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970972142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[2]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.970995036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"[2]R dirty[0..12]: [true, false, false, false, true, false, false, false, true, true, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97101844Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Max found gen is 199","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971039618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Generation requested: 199 >= found:199","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971061216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Next flush: 367","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971082294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971103265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971125745Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:0 gens: 197 197 197","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971149122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:0 flush: 364 364 364 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971173474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971197502Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:0 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971220725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971244409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971272074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971295913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971320278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97134279Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:4 gens: 195 195 195","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971365907Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:4 flush: 359 359 359 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97139014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971414025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:4 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971437087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971460754Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971483911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971507043Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 8 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971530129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971552493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:8 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971575534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:8 flush: 363 363 363 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971602995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971627734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:8 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971651075Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971674533Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971697719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971720876Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 9 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971743947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 9","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971766315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:9 gens: 191 191 191","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971789577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:9 flush: 354 354 354 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971828452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971855372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:9 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971878699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 9","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971902014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971925165Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971952001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 12 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971975764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 12","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.971998259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:12 gens: 193 193 193","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972021401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:12 flush: 357 357 357 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972045676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97206949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:12 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97209254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 12","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972115966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972139065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972162205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 17 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97218511Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 17","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972207425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:17 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972230486Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:17 flush: 363 363 363 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972254709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972281602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:17 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972305583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 17","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972329206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972352329Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972375407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 18 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972398418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 18","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97242063Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:18 gens: 198 195 198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97244366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:18 flush: 366 359 366 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972467571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97249107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:18 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972551068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 18","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97261609Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972646172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972673652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 22 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972697777Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 22","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972720289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:22 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972743416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:22 flush: 363 363 363 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972767787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972791593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:22 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972829835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 22","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972856194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972895458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972941935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 25 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.972988667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 25","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973030383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:25 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973067061Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:25 flush: 363 363 363 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973092938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97312161Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:25 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973145876Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 25","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973169427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97319257Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973215593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 27 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973238711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 27","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973261134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:27 gens: 197 197 197","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973284012Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:27 flush: 364 364 364 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97330813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973331722Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:27 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97335468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 27","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973377985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973401017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973424273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extents 28 dirty","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973450761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 28","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973473794Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:28 gens: 196 196 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973500853Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:28 flush: 363 363 363 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973546022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973593696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:28 dirty: true false true","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973641312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 28","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973676254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973700453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973723726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extent 19 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973747618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 19","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973770039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:19 gens: 198 193 198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973793534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:19 flush: 366 357 366 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973836822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973864382Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:19 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973892572Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973917747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 19","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97394183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973965015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Extent 24 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.973988891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"First source client ID for extent 24","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974011321Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:24 gens: 198 182 198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974034568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:24 flush: 366 338 366 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974058697Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974082367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"extent:24 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974105776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974129672Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"find dest for source 0 for extent at index 24","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974153446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974176493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974204183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974226648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974248155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Found 13 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974269417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Full repair list: {27: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 28: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 18: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 22: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 4: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 17: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 19: ExtentFix { source: ClientId(0), dest: [ClientId(1)] }, 24: ExtentFix { source: ClientId(0), dest: [ClientId(1)] }, 12: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 0: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 9: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 8: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, 25: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974316013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 27, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 27, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 28, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 28, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 18, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 18, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 22, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 22, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 4, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 4, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 17, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 17, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 19, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 19, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 24, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 24, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 12, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 12, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 0, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 0, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 9, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 9, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 8, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 8, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 25, client_id: ClientId(0), flush_number: 367, gen_number: 199 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 25 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 25, source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 25 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974622471Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974652998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974675879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(0)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974697773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974719546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.97474111Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.974762517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.065864233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.06592995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(4)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.065955248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.067916145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.067954413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.067977933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.148955222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.149020247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(8)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.149045047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.151005081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.15104857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.151087391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.227596722Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.227662421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(12)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.22768679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.229926396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.229963759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.229986884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.303822214Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.303887574Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(16)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.303911837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.305522375Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.305560714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.305584372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.38191632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.381977417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(20)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.382002367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.384571422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.384611121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(22)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.384634506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.459271579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.459336808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(24)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.459361763Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.460068896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.460105885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(26)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.460128961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.539815842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.539868123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(28)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.539891962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.540563355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.540620833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(30)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.540668933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.617769918Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.617816491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(32)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.61784085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.619743655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.619784819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(34)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.619809081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.695290859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.695343768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(36)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.6953681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.697189369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.697227429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(38)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.69725131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.777095482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.777141893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(40)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.777165904Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.778895405Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.778933185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(42)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.778955914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(44)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.853934469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(44)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.853979461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(44)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.85400346Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(46)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.85629211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(46)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.856334557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(46)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.856368128Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending flush request ReconciliationId(48)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.931715408Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(48)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.931759172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"skipping flush request ReconciliationId(48)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.93178386Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"no action needed request ReconciliationId(50)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.93347401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(50)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.933516293Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"sending repair request ReconciliationId(50)","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:57.93355893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.012873014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"":"downstairs","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-01-10T18:03:58.01292226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"0","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.012952703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"1","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Transition from Repair to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.012974582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"client":"2","":"downstairs","session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.012996631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.013016742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"b5c145c1-8fd6-4009-8274-4d32fbf32d32 is now active with session: 8682c1e8-4554-4ebe-827b-2f0b5f01de12","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.013038421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957,"session_id":"8682c1e8-4554-4ebe-827b-2f0b5f01de12"} {"msg":"The guest has finished waiting for activation with:199","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:58.013058597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1957} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Read and Verify all blocks (0..600 range:true) Wrote out file "/tmp/test_repair_verify.data" Verify test completed Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Read 100 Done Done Done false 2 Acked 1001 Read 100 Done Done Done false 3 Acked 1002 Read 100 Done Done Done false 4 Acked 1003 Read 100 Done Done Done false 5 Acked 1004 Read 100 Done Done Done false 6 Acked 1005 Read 100 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 6 5 4 3 2 CLIENT: Up:0 ds:6 act:3 ---------------------------------------------------------------- Crucible gen:199 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program